diff --git a/.changeset/bash-parser-service.md b/.changeset/bash-parser-service.md new file mode 100644 index 0000000000..05e5f34208 --- /dev/null +++ b/.changeset/bash-parser-service.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Add an internal bash parsing capability that turns shell command strings into syntax trees, in preparation for per-command permission analysis. No user-facing behavior change yet. diff --git a/AGENTS.md b/AGENTS.md index 2349080c17..26bf9dcdba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ This is a TypeScript monorepo built for agent-assisted development. Keep the roo - `packages/kap-server`: the Kimi Code server, backed by the DI × Scope agent engine (`@moonshot-ai/agent-core-v2`). Exposes sessions over REST + WebSocket (`/api/v1` + `/api/v1/ws`); bootstrapped from `src/start.ts` and consumed by `apps/kimi-code`. The RPC surface is `/api/v1/debug/*` — a reflection dispatcher over the ENTIRE scoped DI registry (every Service callable, no whitelist; `src/transport/registerDebugRoutes.ts` + `serviceDispatcherRoutes.ts`), mounted only with `--debug-endpoints` on a loopback bind and gated by the global bearer auth; repo dev scripts pass the flag. - `packages/klient`: the client SDK — a contract-driven facade over agent-core-v2 with aggregated `global.*` / `session(id).*` / `agent(id).*` methods, zod validation on every call, and klient-level typed event forwarding. Transport is chosen once at creation via subpath entry (`@moonshot-ai/klient/ipc|memory`); both return the same `Klient`. The package also hosts the e2e suites: the legacy `/api/v1` live suites (`test/e2e/legacy/`) and the docker e2e runner (`pnpm --filter @moonshot-ai/klient docker:e2e`). See `packages/klient/AGENTS.md`. - `packages/server-e2e`: live e2e tests and scenarios against a running server (`KIMI_SERVER_URL`, default `http://127.0.0.1:58627`). See `packages/server-e2e/AGENTS.md`. +- `packages/tree-sitter-bash`: a pure-TypeScript bash parser (no runtime deps, no wasm) that produces a syntax tree with tree-sitter-bash 0.25.0 named-node type names and UTF-16 code-unit offsets. `parse(source, { timeoutMs, maxNodes })` runs under a deterministic budget (default 50 ms / 50k nodes, plus per-chain recursion depth caps) and returns a discriminated `ParseResult` (`{ ok, rootNode, hasError }` or `{ ok: false, reason: 'aborted' }`) — callers must treat aborted/hasError trees as "cannot analyze" and degrade. Parser only, no safety judgments; consumers (e.g. Bash tool permission matching) live elsewhere. Known deviations from the reference are tracked in the package README's "Known differences" section, pinned by differential fixtures tested against the real `tree-sitter-bash` wasm (dev-only). ## Environment Requirements diff --git a/flake.nix b/flake.nix index 7f56b35994..5261a5474c 100644 --- a/flake.nix +++ b/flake.nix @@ -77,6 +77,7 @@ ./packages/protocol ./packages/telemetry ./packages/transcript + ./packages/tree-sitter-bash ./apps/kimi-code ./apps/vscode ./apps/kimi-inspect @@ -103,6 +104,7 @@ "@moonshot-ai/protocol" "@moonshot-ai/kimi-telemetry" "@moonshot-ai/transcript" + "@moonshot-ai/tree-sitter-bash" "@moonshot-ai/kimi-code" "kimi-code" "@moonshot-ai/kimi-inspect" @@ -160,7 +162,7 @@ inherit (finalAttrs) pname version src pnpmWorkspaces; inherit pnpm; fetcherVersion = 3; - hash = "sha256-+pzJfoWJwVXIUU8oc56LVpfNjSY6MABID5g11Cm92xw="; + hash = "sha256-niK61WYaqCJLUUlp4AGgq2/DOF3sqst6N9Kpk9noXyE="; }; nativeBuildInputs = [ diff --git a/packages/agent-core-v2/package.json b/packages/agent-core-v2/package.json index caba0689d6..1eb599fdc7 100644 --- a/packages/agent-core-v2/package.json +++ b/packages/agent-core-v2/package.json @@ -63,6 +63,7 @@ "@moonshot-ai/kimi-code-oauth": "workspace:^", "@moonshot-ai/minidb": "workspace:^", "@moonshot-ai/protocol": "workspace:^", + "@moonshot-ai/tree-sitter-bash": "workspace:^", "@mozilla/readability": "^0.6.0", "ajv": "^8.18.0", "ajv-formats": "^3.0.1", diff --git a/packages/agent-core-v2/scripts/check-domain-layers.mjs b/packages/agent-core-v2/scripts/check-domain-layers.mjs index b93a586984..cb2cfc7685 100644 --- a/packages/agent-core-v2/scripts/check-domain-layers.mjs +++ b/packages/agent-core-v2/scripts/check-domain-layers.mjs @@ -102,6 +102,11 @@ const DOMAIN_LAYER = new Map([ // Depends only on `_base`; sits in L1 beside the other program-control // layer substrates. ['task', 1], + // `bashParser` is the App-scope adapter over the pure + // `@moonshot-ai/tree-sitter-bash` package (bash source → syntax tree DTO). + // It injects no services, so it sits in L1 beside the other pure + // capabilities. + ['bashParser', 1], // persistence/ and os/ — the two-level scopes. `interface` holds contracts // (same layer as the old domains they replace); `backends` holds // implementations that may depend on cross-domain services at various layers. diff --git a/packages/agent-core-v2/src/app/bashParser/bashParser.ts b/packages/agent-core-v2/src/app/bashParser/bashParser.ts new file mode 100644 index 0000000000..154c8550f2 --- /dev/null +++ b/packages/agent-core-v2/src/app/bashParser/bashParser.ts @@ -0,0 +1,42 @@ +/** + * `bashParser` domain (L1) — bash source parsing capability. + * + * Defines the `IBashParserService` that parses a bash source string into a + * syntax tree through the pure `@moonshot-ai/tree-sitter-bash` package, plus + * the wire-safe DTO types it returns: `BashSyntaxNode` drops the cyclic + * `parent` link so results can cross the RPC boundary, and offsets are + * UTF-16 code units (`text` always equals `source.slice(start, end)`). The + * parse runs under a deterministic budget — budget exhaustion yields + * `{ ok: false, reason: 'aborted' }` and malformed input yields + * `hasError: true`, never a throw; callers that cannot analyze a command + * must degrade on either signal. Bound at App scope. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; + +export interface BashSyntaxNode { + readonly type: string; + readonly text: string; + readonly startIndex: number; + readonly endIndex: number; + readonly isNamed: boolean; + readonly children: readonly BashSyntaxNode[]; +} + +export interface BashParseOptions { + readonly timeoutMs?: number; + readonly maxNodes?: number; +} + +export type BashParseResult = + | { readonly ok: true; readonly hasError: boolean; readonly root: BashSyntaxNode } + | { readonly ok: false; readonly reason: 'aborted' }; + +export interface IBashParserService { + readonly _serviceBrand: undefined; + + parse(source: string, options?: BashParseOptions): BashParseResult; +} + +export const IBashParserService: ServiceIdentifier = + createDecorator('bashParserService'); diff --git a/packages/agent-core-v2/src/app/bashParser/bashParserService.ts b/packages/agent-core-v2/src/app/bashParser/bashParserService.ts new file mode 100644 index 0000000000..60c06bcbba --- /dev/null +++ b/packages/agent-core-v2/src/app/bashParser/bashParserService.ts @@ -0,0 +1,49 @@ +/** + * `bashParser` domain (L1) — `IBashParserService` implementation. + * + * Thin adapter over the pure `@moonshot-ai/tree-sitter-bash` package: runs + * its budgeted `parse` and snapshots the returned tree into the wire-safe + * `BashSyntaxNode` DTO (source-ordered children including anonymous tokens, + * `parent` links dropped). Owns no state and injects no services. Bound at + * App scope. + */ + +import { parse } from '@moonshot-ai/tree-sitter-bash'; +import type { SyntaxNode } from '@moonshot-ai/tree-sitter-bash'; + +import { InstantiationType } from '#/_base/di/extensions'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; + +import type { BashParseOptions, BashParseResult, BashSyntaxNode } from './bashParser'; +import { IBashParserService } from './bashParser'; + +function snapshot(node: SyntaxNode): BashSyntaxNode { + return { + type: node.type, + text: node.text, + startIndex: node.startIndex, + endIndex: node.endIndex, + isNamed: node.isNamed, + children: node.children.map(snapshot), + }; +} + +export class BashParserService implements IBashParserService { + declare readonly _serviceBrand: undefined; + + parse(source: string, options: BashParseOptions = {}): BashParseResult { + const result = parse(source, options); + if (!result.ok) { + return { ok: false, reason: result.reason }; + } + return { ok: true, hasError: result.hasError, root: snapshot(result.rootNode) }; + } +} + +registerScopedService( + LifecycleScope.App, + IBashParserService, + BashParserService, + InstantiationType.Delayed, + 'bashParser', +); diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index a6678675b3..99a67fda31 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -282,6 +282,8 @@ export * from '#/app/workspaceRegistry/workspacePersistence'; export * from '#/app/workspaceRegistry/fileWorkspacePersistence'; import '#/app/workspaceRegistry/workspaceQueryService'; import '#/app/git/gitService'; +export * from '#/app/bashParser/bashParser'; +import '#/app/bashParser/bashParserService'; export * from '#/session/process/processRunner'; export * from '#/session/process/processRunnerService'; export * from '#/session/sessionFs/errors'; diff --git a/packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts b/packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts new file mode 100644 index 0000000000..2228f50cac --- /dev/null +++ b/packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts @@ -0,0 +1,72 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { DisposableStore } from '#/_base/di/lifecycle'; +import { createServices, type TestInstantiationService } from '#/_base/di/test'; +import type { BashSyntaxNode } from '#/app/bashParser/bashParser'; +import { IBashParserService } from '#/app/bashParser/bashParser'; +import { BashParserService } from '#/app/bashParser/bashParserService'; + +describe('BashParserService', () => { + let disposables: DisposableStore; + let ix: TestInstantiationService; + let service: IBashParserService; + + beforeEach(() => { + disposables = new DisposableStore(); + ix = createServices(disposables, { + additionalServices: (reg) => { + reg.define(IBashParserService, BashParserService); + }, + }); + service = ix.get(IBashParserService); + }); + + afterEach(() => { + disposables.dispose(); + }); + + it('splits a compound command into per-command nodes', () => { + const result = service.parse('git status && rm -rf /'); + if (!result.ok) { + throw new Error('expected ok'); + } + expect(result.hasError).toBe(false); + expect(result.root.type).toBe('program'); + const commands: string[] = []; + const walk = (node: BashSyntaxNode): void => { + if (node.type === 'command') { + commands.push(node.text); + } + node.children.forEach(walk); + }; + walk(result.root); + expect(commands).toEqual(['git status', 'rm -rf /']); + }); + + it('flags malformed input with hasError instead of throwing', () => { + const result = service.parse('echo "unterminated'); + if (!result.ok) { + throw new Error('expected ok'); + } + expect(result.hasError).toBe(true); + }); + + it('reports budget exhaustion as aborted', () => { + const result = service.parse('ls; '.repeat(20000), { maxNodes: 100 }); + expect(result).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('returns a JSON-serializable tree with text/range fidelity', () => { + const source = 'echo "你好 🎉" | grep 你'; + const result = service.parse(source); + if (!result.ok) { + throw new Error('expected ok'); + } + const roundTripped = JSON.parse(JSON.stringify(result.root)) as BashSyntaxNode; + const check = (node: BashSyntaxNode): void => { + expect(node.text).toBe(source.slice(node.startIndex, node.endIndex)); + node.children.forEach(check); + }; + check(roundTripped); + }); +}); diff --git a/packages/tree-sitter-bash/README.md b/packages/tree-sitter-bash/README.md new file mode 100644 index 0000000000..8ef128d62f --- /dev/null +++ b/packages/tree-sitter-bash/README.md @@ -0,0 +1,273 @@ +# @moonshot-ai/tree-sitter-bash + +A pure-TypeScript bash parser that produces a syntax tree whose named node +types match [tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash) +0.25.0 one-to-one, built for agent-side command permission analysis. + +- No native addons; offsets are UTF-16 code units (`node.text` is always a + direct `source.slice(startIndex, endIndex)`). +- Parsing runs under a hard budget and never throws: budget exhaustion + returns `{ ok: false, reason: 'aborted' }`, malformed input returns a + degraded tree with `hasError: true`. +- Correctness is enforced by a differential test suite that parses hundreds + of samples — plus the complete official tree-sitter-bash 0.25.0 corpus — + with both this parser and the real tree-sitter-bash (wasm) and compares + the trees byte-for-byte (see `test/fixtures/`). The only remaining + deviations are the documented ones listed under + [Known differences](#known-differences-from-tree-sitter-bash) below. + +## API + +```ts +import { parse } from '@moonshot-ai/tree-sitter-bash'; + +const result = parse('git status && rm -rf /'); +if (result.ok) { + // result.rootNode: program → list → command … +} +``` + +- `parse(source: string, options?: ParseOptions): ParseResult` +- `ParseResult` is one of: + - `{ ok: true; rootNode: SyntaxNode; hasError: boolean }` + - `{ ok: false; reason: 'aborted' }` +- `ParseOptions`: `{ timeoutMs?: number; maxNodes?: number }` (defaults + 50 ms / 50 000 nodes; `timeoutMs: Infinity` disables the time check). +- `SyntaxNode`: `{ type, text, startIndex, endIndex, isNamed, parent, + children, namedChildren }`. Named node types come from tree-sitter-bash's + `node-types.json`; `children` includes the anonymous (punctuation / + keyword token) nodes as well, `namedChildren` only the named ones, and + `descendantsOfType(root, ...types)` walks named descendants. +- Depth caps (pathological nesting degrades locally — ERROR nodes, + `hasError: true` — instead of overflowing the stack): + - `MAX_SUBSTITUTION_DEPTH = 150` for the $( … ) / `…` / <( … ) + sub-parser chain (the deepest per level, ~13 frames — measured stack + overflow at ~380–500 levels on a default Node stack, so the cap keeps + a ≥2.5× margin); + - `MAX_PARSE_DEPTH = 500` for subshell/compound/`${…}`/expression + nesting (verified safe at those caps); + - `MAX_SCAN_DEPTH = 1024` for the lexer's own scan recursion. + +## Budget semantics + +The budget caps total work, not input size: a multi-hundred-KB string or +heredoc body parses fine (it produces only a handful of nodes), but a +source whose tree would exceed 50 000 nodes — a megabyte-long `case` +statement, tens of thousands of arithmetic expressions — is aborted under +the defaults. Every created node ticks the budget; long character-level +scan loops re-check the deadline periodically. When either limit is hit, +`parse` returns `{ ok: false, reason: 'aborted' }` and the abort is prompt +(see Performance). Pass `timeoutMs` / `maxNodes` to raise the limits. + +## Error recovery + +Malformed input never throws either. Unterminated constructs (quotes, +expansions, substitutions, heredocs, compound commands) are kept as +partial nodes and flagged with `hasError: true`; tokens that cannot start +or continue a statement (stray `)`, a leading `&&`, …) are wrapped in +`ERROR` nodes and parsing continues. Newlines never appear in the tree: +statement-terminating `\n`s produce no nodes at all (matching +tree-sitter-bash, whose scanner only emits `\n` tokens in heredoc +position), while `;` / `&` terminators are anonymous children. As a +last-resort guard against parser bugs, any unexpected internal exception +degrades to a `program` root with a single `ERROR` child spanning the +source and `hasError: true` — callers still get a usable tree. + +## Known differences from tree-sitter-bash + +Named node types always come from tree-sitter-bash's `node-types.json`, but +for the following constructs the tree shape deliberately deviates from what +tree-sitter-bash 0.25.0 produces (each verified against the real parser, +and pinned by a stored expectation in `test/fixtures/` — see +`test/helpers/known-differences.ts`): + +- `<>` (read-write redirect) is parsed as a normal `file_redirect` + operator; tree-sitter-bash 0.25.0 fails to parse it. +- Several heredocs on one line (`cat <`) is a + `word` in the reference even when its content would make it a glob + (`cmake_modules` — a scanner-state quirk); this parser classifies + patterns by content, so such a pattern is an `extglob_pattern` here. A + continuation INSIDE a pattern (`cont\ued)`) is an error in the + reference (word "cont" + `ERROR` "ued", `hasError: true`), and a + continuation fused with a digit (`a\1)`) is an + `extglob_pattern` there; this parser keeps the continuation and parses + the whole pattern cleanly as one `word` — which is also how bash itself + reads it. +- A comparison right side with TWO substitutions or quotes + (`*${x}*${y}`, `*"s"*"t"`) does not fit the reference's one-construct + pattern rule either; it recovers with a nested `binary_expression` + there, while this parser keeps a single `concatenation` right side. +- A few `[[ … ]]` comparison right-hand sides deviate: + - A negative decimal operand (`[[ $n == -0.5 ]]`) is a + `unary_expression` (`-` over word "0.5") in the reference; this parser + produces a single `extglob_pattern` "-0.5" (the unary-minus reading + only covers integers here). + - A test operator fused with an expansion (`[[ -x$f && … ]]`) is a + `unary_expression` (`-` over a `concatenation` of `x` and `$f`) in the + reference; this parser produces a flat `concatenation` of `-x` and + `$f` with no unary wrapper. + - An escaped `|` inside a pattern (`[[ $a == foo\|bar ]]`) splits the + expression in the reference — `binary_expression(==, extglob "foo\")` + then `|` then word "bar" (a reference quirk); this parser keeps one + `extglob_pattern` "foo\|bar". +- `${!# }` and `${!## }` (and `${!##/}`) are pathological expansion forms: + the reference recovers with zero-width operator tokens, this parser with + a flagged partial `expansion`; the shapes differ. (The sane forms + `${#}`, `${!#}`, `${!##}`, `${#!}` all match exactly.) +- A base prefix fused with an expansion (`10#${x}`) is one `number` node + spanning the expansion in the reference (a quirk); this parser produces + a `concatenation` of word "10#" and the expansion. +- `${v:-(default)}` (a parenthesized default value) is a plain `word` + here; the reference parses it as an `array` node (a quirk). +- `${=1}` (zsh-style word splitting flag) produces a `variable_name` "1" + in the reference; this parser produces a `word`. +- An escaped space or tab between arguments is dropped by the reference + as invisible extras (its tree has gaps: `echo 1 \ 2` produces number + "1" and number "2", and `echo a\b` splits into word "a" and word + "b"); this parser keeps the escape inside the word. (An escaped space + INSIDE a word, `a\ b`, is kept by both parsers and matches.) +- A `$` directly fused with a backtick substitution (`` $`echo x` ``) is + one `command_substitution` with a `$`+backtick token in the reference; + this parser produces a `($)` token followed by the substitution. +- A non-ASCII “identifier” in assignment position (`变量=值`, `é=1`) is a + `variable_assignment` flagged `hasError` in the reference; this parser + keeps it a plain command word — which is also how bash itself treats it + (variable names are ASCII-only, so the text runs as a command). +- A negative literal after a compound assignment in a c-style for header + (`for (( … j *= -1, … ))`) is folded into a `number` "-1" by the + reference; this parser produces a `unary_expression` (`-` over number + "1") — matching what the reference itself produces everywhere else. +- A double backslash in a replacement value (`${x// /\\|}`) loses its + first backslash in the reference's tree (a quirk — the `word` starts + one character later); this parser keeps both. +- `string_content` is not split at newlines (tree-sitter-bash's scanner + splits it). + +For completeness, a few constructs that LOOK like deviations but are not — +the reference behaves the same way (verified): + +- `coproc` has no grammar support in tree-sitter-bash 0.25.0 and parses as + an ordinary command; so does this parser. +- `select` is parsed as a `for_statement` with a `select` keyword token — + that is exactly the reference's tree (there is no `select_statement` + node type). +- `$"…"` (translated string): in command-argument position it is an + anonymous `$` argument followed by the `string` argument; everywhere + else (assignment values, redirect destinations, for-loop values, …) it + is a `translated_string` node. Both match the reference exactly. + `$'…'` is a single-child-free `ansi_c_string` node, also matching. +- Brace expansion: only `{N..M}` (unsigned digits) is a `brace_expression`; + `{a..z}`, `{1..10..2}` and `{-5..5}` are plain word concatenations in the + reference and here. +- Statement-position `((word++…))` (inner text starting with a bare + identifier plus a postfix `++`/`--`) is a `test_command` in the + reference — its statement-position arithmetic grammar does not accept + that form — while `((x = 1))`, `((b + a++))`, `((a[i]++))` and + argument-position `((a++))` are `arithmetic_expansion`. This parser + matches all of these, including after `!`. +- Command-prefix redirects take exactly one destination (`> a cmd x` makes + `cmd` the `command_name`), while redirects after the command name consume + every following word (`cmd > out arg` puts both words in the redirect) — + this matches tree-sitter-bash's actual disambiguation. + +## Performance + +Measured on an Apple-silicon MacBook (Node 24, default 50 ms / 50 000-node +budget): + +- A typical one-line command (`git status && rm -rf /`): ~4 µs per parse. +- A 100 KB realistic deployment script (functions, loops, redirects, + expansions, heredocs): tens of milliseconds per parse (~20–60 ms across + runs and machines) — the same order of magnitude as the default 50 ms + budget, so such inputs may return `ok: true` or a prompt abort depending + on the machine; raise `timeoutMs` when parsing whole scripts. +- A 500 KB heredoc body: parses within the default budget (the tree has + only a handful of nodes). +- The abort path is prompt: a 400 KB node-budget bomb aborts in ~20 ms + (hard guarantee asserted by the test suite: < 100 ms). + +These numbers are smoke-tested as orders of magnitude in +`test/performance.test.ts` to guard against accidental quadratic +complexity; absolute timings vary by machine. diff --git a/packages/tree-sitter-bash/package.json b/packages/tree-sitter-bash/package.json new file mode 100644 index 0000000000..dab858aa31 --- /dev/null +++ b/packages/tree-sitter-bash/package.json @@ -0,0 +1,47 @@ +{ + "name": "@moonshot-ai/tree-sitter-bash", + "version": "0.1.0", + "private": true, + "description": "A pure-TypeScript bash parser producing a syntax tree whose named node types match tree-sitter-bash one-to-one.", + "license": "MIT", + "author": "Moonshot AI", + "homepage": "https://github.com/MoonshotAI/kimi-code/tree/main/packages/tree-sitter-bash#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/MoonshotAI/kimi-code.git", + "directory": "packages/tree-sitter-bash" + }, + "bugs": { + "url": "https://github.com/MoonshotAI/kimi-code/issues" + }, + "keywords": [ + "kimi", + "bash", + "shell", + "parser", + "tree-sitter" + ], + "files": [ + "dist" + ], + "type": "module", + "imports": { + "#/*": "./src/*.ts" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsdown", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "clean": "rm -rf dist" + }, + "devDependencies": { + "tree-sitter-bash": "0.25.0", + "web-tree-sitter": "0.25.10" + } +} diff --git a/packages/tree-sitter-bash/src/budget.ts b/packages/tree-sitter-bash/src/budget.ts new file mode 100644 index 0000000000..35ffe9db76 --- /dev/null +++ b/packages/tree-sitter-bash/src/budget.ts @@ -0,0 +1,69 @@ +// src/budget.ts +// +// Parse budget: a hard cap on wall-clock time and on the number of syntax +// nodes a single parse may create. The parser calls `budget.tick()` every +// time it creates a node; long scan loops (word runs, strings, heredoc +// bodies) call `budget.progress()` periodically so a pathological single +// token still hits the deadline without inflating the node count. When +// either limit is exceeded the methods throw `Aborted`, which the `parse` +// entry point catches and reports as `{ ok: false, reason: 'aborted' }`. + +export const DEFAULT_TIMEOUT_MS = 50; +export const DEFAULT_MAX_NODES = 50_000; + +/** Internal control-flow error. Never escapes the `parse` entry point. */ +export class Aborted extends Error { + constructor(message: string) { + super(message); + this.name = 'Aborted'; + } +} + +export interface BudgetOptions { + /** Wall-clock limit in milliseconds. `Infinity` disables the time check. */ + timeoutMs?: number; + /** Maximum number of nodes the parse may create. */ + maxNodes?: number; +} + +export class ParseBudget { + private readonly deadline: number; + private readonly maxNodes: number; + private nodeCount = 0; + + constructor(options: BudgetOptions = {}) { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.deadline = Date.now() + timeoutMs; + this.maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES; + } + + /** Number of nodes created so far. */ + get nodesUsed(): number { + return this.nodeCount; + } + + /** + * Account for one created node and re-check the deadline. Throws `Aborted` + * when the node cap is exceeded or the deadline has been reached. + */ + tick(): void { + this.nodeCount++; + if (this.nodeCount > this.maxNodes) { + throw new Aborted(`parse aborted: node budget exceeded (${this.nodeCount} > ${this.maxNodes})`); + } + if (Date.now() >= this.deadline) { + throw new Aborted(`parse aborted: timeout`); + } + } + + /** + * Re-check the deadline WITHOUT counting a node. For long scan loops that + * run many iterations per produced node (character-level scanning), called + * at intervals so the deadline is still enforced promptly. + */ + progress(): void { + if (Date.now() >= this.deadline) { + throw new Aborted(`parse aborted: timeout`); + } + } +} diff --git a/packages/tree-sitter-bash/src/grammar.ts b/packages/tree-sitter-bash/src/grammar.ts new file mode 100644 index 0000000000..8a16268490 --- /dev/null +++ b/packages/tree-sitter-bash/src/grammar.ts @@ -0,0 +1,134 @@ +// src/grammar.ts +// +// Static grammar tables for bash: operator tables, keyword sets and +// variable-name sets shared by the lexer and the parser. + +/** Single-character special parameters: $@ $* $# $? $- $$ $! $0 and $_. */ +export const SPECIAL_VARIABLES = ['@', '*', '#', '?', '-', '$', '!', '0', '_'] as const; + +/** Special parameter characters that may follow `$` directly, as a string + * for `includes` checks. Derived from SPECIAL_VARIABLES by dropping `0` + * and `_`, which are word characters matched by the `\w+` rule instead. */ +export const SPECIAL_VARIABLE_CHARS = SPECIAL_VARIABLES.filter((ch) => !/\w/.test(ch)).join(''); + +/** Operators that open a `file_redirect` (heredoc and herestring operators + * are handled separately). `<>` is included even though tree-sitter-bash + * 0.25.0 fails to parse it — it is a real bash operator (`exec 3<>file`). */ +export const FILE_REDIRECT_OPERATORS = ['<', '>', '>>', '>&', '<&', '&>', '&>>', '>|', '<>', '>&-', '<&-'] as const; + +/** Keywords that open a declaration_command at statement position. */ +export const DECLARATION_COMMAND_KEYWORDS = ['declare', 'typeset', 'export', 'readonly', 'local'] as const; + +/** Keywords that open an unset_command at statement position. */ +export const UNSET_COMMAND_KEYWORDS = ['unset', 'unsetenv'] as const; + +/** Reserved words that may not be used as a function name in the + * `name() { ...; }` form (they are keywords in command position). */ +export const RESERVED_WORDS = [ + 'if', + 'then', + 'elif', + 'else', + 'fi', + 'while', + 'until', + 'do', + 'done', + 'for', + 'select', + 'in', + 'case', + 'esac', + 'function', + '{', + '}', + '!', + ...DECLARATION_COMMAND_KEYWORDS, + ...UNSET_COMMAND_KEYWORDS, +] as const; + +/** Infix operators inside arithmetic / test expressions with their + * tree-sitter-bash precedence level (grammar.js PREC table; higher binds + * tighter). `test_operator` binaries sit at level 10 (TEST) and are handled + * separately. Ternary `? :` is level 2, postfix `++`/`--` level 18, prefix + * `++`/`--` level 17, prefix `!`/`~`/`+`/`-` level 11. */ +export const EXPRESSION_PRECEDENCE: Readonly> = { + '+=': 0, + '-=': 0, + '*=': 0, + '/=': 0, + '%=': 0, + '**=': 0, + '<<=': 0, + '>>=': 0, + '&=': 0, + '^=': 0, + '|=': 0, + '=': 1, + '=~': 1, + '||': 3, + '&&': 4, + '|': 5, + '^': 6, + '&': 7, + '==': 8, + '!=': 8, + '<': 9, + '<=': 9, + '>': 9, + '>=': 9, + '<<': 12, + '>>': 12, + '+': 13, + '-': 13, + '*': 14, + '/': 14, + '%': 14, + '**': 15, +}; + +/** All operator texts the expression lexer recognizes, longest first so + * matching is greedy (`**=` before `**` before `*`). `;` and `,` only act + * as separators (c-style for headers and comma lists), not as operators. */ +export const EXPRESSION_OPERATORS = [ + '**=', + '<<=', + '>>=', + '++', + '--', + '**', + '<<', + '>>', + '<=', + '>=', + '==', + '!=', + '&&', + '||', + '+=', + '-=', + '*=', + '/=', + '%=', + '&=', + '^=', + '|=', + '=~', + '=', + '+', + '-', + '*', + '/', + '%', + '^', + '&', + '|', + '<', + '>', + '!', + '~', + '?', + ':', + ',', + ';', +] as const; diff --git a/packages/tree-sitter-bash/src/index.ts b/packages/tree-sitter-bash/src/index.ts new file mode 100644 index 0000000000..2737d6529e --- /dev/null +++ b/packages/tree-sitter-bash/src/index.ts @@ -0,0 +1,4 @@ +export * from './node'; +export * from './budget'; +export * from './parse'; +export * from './grammar'; diff --git a/packages/tree-sitter-bash/src/lexer.ts b/packages/tree-sitter-bash/src/lexer.ts new file mode 100644 index 0000000000..807deb7ee9 --- /dev/null +++ b/packages/tree-sitter-bash/src/lexer.ts @@ -0,0 +1,629 @@ +// src/lexer.ts +// +// Hand-written bash tokenizer. Produces a flat token stream of: +// +// word a maximal run of adjacent word material: bare characters, +// '...' / "..." quotes, $var / ${...} / $(...) / `...` expansions, +// <(...) / >(...) process substitutions. Quotes and substitutions +// are skipped over with nesting awareness; the parser re-scans +// the token range to build the actual sub-tree. The single +// characters { } [ ] are emitted as their own word tokens (they +// are not word characters in bash), matching tree-sitter-bash's +// _special_character behaviour. +// op statement operators (&& || | |& ; ;; & ( )) and redirect +// operators (< > >> >& <& &> &>> >| <> >&- <&- <<< << <<-). +// io_number a run of digits immediately followed by < or > — the file +// descriptor prefix of a redirect (2>/dev/null). +// newline a statement-terminating \n. Carries any heredoc bodies that +// were queued on the lexer and are scanned right after the line. +// comment # ... to end of line (only at token start; # inside a word is +// a plain word character). +// eof end of the lexer's range. Also carries pending heredoc bodies. +// +// Additionally: +// - `;&` and `;;&` are single op tokens (case_item fallthrough +// terminators). +// - `((...))` at token start is scanned as one word token (arithmetic +// command / c-style for header), so the parser can re-scan the range +// with its expression parser; the word ends right after the balanced +// close and does not merge with following word characters. +// - peekAt(n) looks arbitrarily far ahead (function_definition +// detection); reposition(pos) rewinds the stream (test commands parse +// their [[ ... ]] range character-wise and then resume tokenizing past +// the closer). +// +// The lexer is range-bounded: sub-parsers lex $(...) / `...` bodies with +// their own Lexer over the same source but a narrower [start, end) window. +// +// Heredoc queue: the parser registers a HeredocSpec for every << / <<- it +// accepts. When the lexer produces the next newline (or eof) token it scans +// one body per queued spec, in registration order, and attaches them to the +// token; the parser completes the matching heredoc_redirect nodes when it +// consumes that token. +// +// Budget: the lexer never creates nodes, so it only checks the deadline — +// once per produced token and periodically (every SCAN_TICK_INTERVAL +// characters) inside every long scan loop (word runs, quote/paren skipping, +// comments, blanks, heredoc bodies) — via budget.progress(), so a +// pathological single token cannot starve the deadline check. Node counting +// (budget.tick()) is the parser's job. + +import type { ParseBudget } from '#/budget'; +import { SPECIAL_VARIABLE_CHARS } from '#/grammar'; + +export type TokenType = 'word' | 'op' | 'io_number' | 'newline' | 'comment' | 'eof'; + +export interface Token { + readonly type: TokenType; + readonly start: number; + readonly end: number; + /** Heredoc bodies scanned when this newline/eof token was produced. */ + readonly heredocBodies: HeredocBody[]; +} + +export interface HeredocSpec { + /** Delimiter text with any quoting removed — what must appear at line start. */ + readonly delimiter: string; + /** `<<-`: leading tabs of the first body line are stripped. */ + readonly stripTabs: boolean; + /** The delimiter was quoted ('EOF', "EOF", \EOF): the body is not expanded. */ + readonly quoted: boolean; +} + +export interface HeredocBody { + /** Start of the heredoc_body node (after first-line tab stripping). */ + readonly bodyStart: number; + /** End of the heredoc_body node: right before the end marker, so the final + * newline and any tabs preceding the marker stay inside the body. */ + readonly bodyEnd: number; + /** Start of the heredoc_end marker (the delimiter word itself). */ + readonly endStart: number; + /** End of the heredoc_end marker. */ + readonly endEnd: number; + /** False when the delimiter line never appeared: body runs to the end of + * the lexer's range and there is no heredoc_end. */ + readonly found: boolean; +} + +/** How often long scan loops tick the budget, in scanned characters. */ +const SCAN_TICK_INTERVAL = 2048; + +/** + * Depth cap for the scanBalanced ↔ skipDoubleQuoted mutual recursion + * (`${a#"${a#"…` nests two scan frames per level). Beyond the cap the scan + * reports "unbalanced", letting the parser degrade the construct with + * hasError instead of overflowing the call stack. Sized above the parser's + * own nesting guards (which trigger first for parser-driven recursion); + * this cap only protects lexer-driven scans of pathological input. + */ +const MAX_SCAN_DEPTH = 1024; + +const CONTROL_OPERATORS = ['&>>', '&>', '&&', '&', '|&', '||', '|', ';;&', ';;', ';&', ';', '(', ')'] as const; +const REDIRECT_OPERATORS = ['<<-', '<<<', '<<', '<&-', '<&', '<>', '<', '>&-', '>&', '>>', '>|', '>'] as const; + +function isWordChar(ch: string | undefined): boolean { + return ch !== undefined && /[\w]/.test(ch); +} + +function isBlank(ch: string | undefined): boolean { + return ch === ' ' || ch === '\t' || ch === '\r'; +} + +function isDigitAt(source: string, i: number): boolean { + const ch = source[i]!; + return ch >= '0' && ch <= '9'; +} + +/** Skip a "..." quoted region starting at `i` (which points at the opening + * quote). Returns the index just past the closing quote, or `end` when the + * string is unterminated. Substitution-aware: $(...), ${...} and `...` + * inside the string may themselves contain quotes. `depth` tracks the + * scanBalanced ↔ skipDoubleQuoted recursion (see MAX_SCAN_DEPTH). */ +export function skipDoubleQuoted(source: string, budget: ParseBudget, i: number, end: number, depth = 0): number { + if (depth >= MAX_SCAN_DEPTH) return end; + let j = i + 1; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') return j + 1; + if (ch === '`') { + j = skipBacktick(source, budget, j, end); + continue; + } + if (ch === '$') { + const next = source[j + 1]; + if (next === '(') { + j = scanBalancedStatements(source, budget, j + 1, end, depth + 1).end; + continue; + } + if (next === '{') { + j = scanBalanced(source, budget, j + 1, end, '{', '}', depth + 1).end; + continue; + } + } + j++; + } + return end; +} + +/** Skip a '...' region starting at `i`. No escapes exist in raw strings. */ +export function skipSingleQuoted(source: string, _budget: ParseBudget, i: number, end: number): number { + const close = source.indexOf("'", i + 1); + if (close === -1 || close >= end) return end; + return close + 1; +} + +/** Skip a `...` region starting at `i`; \` is an escaped backtick. */ +export function skipBacktick(source: string, budget: ParseBudget, i: number, end: number): number { + let j = i + 1; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '`') return j + 1; + j++; + } + return end; +} + +/** Result of scanning a balanced region. */ +export interface BalancedScan { + /** Index just past the matching close, or `end` when unbalanced. */ + end: number; + /** Whether the matching close was actually found. */ + balanced: boolean; +} + +/** Scan a balanced open/close region starting at `i` (which points at the + * opening character). Quote- and escape-aware. `depth` tracks the + * scanBalanced ↔ skipDoubleQuoted recursion (see MAX_SCAN_DEPTH). */ +export function scanBalanced( + source: string, + budget: ParseBudget, + i: number, + end: number, + open: string, + close: string, + depth = 0, +): BalancedScan { + if (depth >= MAX_SCAN_DEPTH) return { end, balanced: false }; + let nesting = 0; + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === open) { + nesting++; + } else if (ch === close) { + nesting--; + if (nesting === 0) return { end: j + 1, balanced: true }; + } else if (ch === '"') { + j = skipDoubleQuoted(source, budget, j, end, depth + 1); + continue; + } else if (ch === "'") { + j = skipSingleQuoted(source, budget, j, end); + continue; + } else if (ch === '`') { + j = skipBacktick(source, budget, j, end); + continue; + } + j++; + } + return { end, balanced: false }; +} + +/** Words after which a `case` word opens a case_statement (statement + * position) rather than being an ordinary argument. */ +const CASE_ENABLING_WORDS: ReadonlySet = new Set(['if', 'then', 'elif', 'else', 'while', 'until', 'do']); + +/** + * Case-aware variant of scanBalanced for `(` … `)` regions that may hold + * full statements (command/process substitution bodies). A naive paren + * count closes the region early on a case_item pattern (`a) 1;; esac`): + * every item's `)` would decrement the depth. While a `case` is open (seen + * in statement position, not yet closed by `esac`), a `)` at the paren + * depth where the `case` started is a pattern close and does NOT count; + * parens from other constructs inside the case (subshells, $(…), the + * optional `(` of an item) balance themselves normally. + * + * Statement position is approximated lexically: `case` counts as a keyword + * after `(`, `;`, `&`, `|`, a newline, `{`, or one of the compound + * keywords in CASE_ENABLING_WORDS — so `echo case` does not confuse the + * scan. `esac` pops the innermost open case regardless of position (the + * reference scanner emits the esac token even in argument position). + */ +export function scanBalancedStatements( + source: string, + budget: ParseBudget, + i: number, + end: number, + depth = 0, +): BalancedScan { + if (depth >= MAX_SCAN_DEPTH) return { end, balanced: false }; + let nesting = 0; + /** Paren depths at which each open case_statement started. */ + const caseDepths: number[] = []; + let j = i; + /** What preceded the current position: 'start' | 'sep' | 'keyword' | 'word'. */ + let previous: 'start' | 'sep' | 'keyword' | 'word' = 'start'; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(source, budget, j, end, depth + 1); + previous = 'word'; + continue; + } + if (ch === "'") { + j = skipSingleQuoted(source, budget, j, end); + previous = 'word'; + continue; + } + if (ch === '`') { + j = skipBacktick(source, budget, j, end); + previous = 'word'; + continue; + } + if (ch === ' ' || ch === '\t' || ch === '\r') { + j++; + continue; + } + if (ch === '\n' || ch === ';' || ch === '&' || ch === '|') { + previous = 'sep'; + j++; + continue; + } + if (ch === '{') { + previous = 'sep'; + j++; + continue; + } + if (ch === '(') { + nesting++; + previous = 'sep'; + j++; + continue; + } + if (ch === ')') { + if (caseDepths.length > 0 && caseDepths.at(-1) === nesting) { + // A case_item pattern close: not a paren. + previous = 'sep'; + j++; + continue; + } + nesting--; + if (nesting === 0) return { end: j + 1, balanced: true }; + previous = 'word'; + j++; + continue; + } + // A word run: check for the case/esac keywords. + let k = j; + while (k < end && isWordChar(source[k])) k++; + if (k > j) { + const word = source.slice(j, k); + if (word === 'esac' && caseDepths.length > 0) { + caseDepths.pop(); + previous = 'sep'; + } else if (word === 'case' && previous !== 'word') { + caseDepths.push(nesting); + previous = 'word'; + } else { + previous = CASE_ENABLING_WORDS.has(word) ? 'keyword' : 'word'; + } + j = k; + continue; + } + previous = 'word'; + j++; + } + return { end, balanced: false }; +} + +/** Skip a $-construct starting at `i` (which points at the `$`). Handles + * $(...), $((...)), ${...}, $'...' (escape-aware: \' does not close), + * $name and the single-character specials. A `$` followed by anything else + * (including a double quote) consumes just the `$`. */ +export function skipDollar(source: string, budget: ParseBudget, i: number, end: number): number { + const next = source[i + 1]; + if (next === '(') return scanBalancedStatements(source, budget, i + 1, end).end; + if (next === '{') return scanBalanced(source, budget, i + 1, end, '{', '}').end; + // $[...] legacy arithmetic (only when `[` directly follows the `$`; `$a[0]` + // takes the word-character branch above). + if (next === '[') return scanBalanced(source, budget, i + 1, end, '[', ']').end; + if (next === "'") { + // ANSI-C string: \' is an escaped quote and does not terminate it. + let j = i + 2; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === "'") return j + 1; + j++; + } + return end; + } + if (isWordChar(next)) { + let j = i + 1; + while (j < end && isWordChar(source[j])) j++; + return j; + } + if (next !== undefined && SPECIAL_VARIABLE_CHARS.includes(next)) return i + 2; + return i + 1; +} + +export class Lexer { + /** Current scan position; exposed for the parser's heredoc bookkeeping. */ + pos: number; + private lookahead: Token[] = []; + private readonly pendingHeredocs: HeredocSpec[] = []; + + constructor( + private readonly source: string, + private readonly budget: ParseBudget, + start = 0, + private readonly end = source.length, + ) { + this.pos = start; + } + + /** End of the lexer's range (exclusive). */ + get rangeEnd(): number { + return this.end; + } + + /** Queue a heredoc body to be scanned after the current line. */ + queueHeredoc(spec: HeredocSpec): void { + this.pendingHeredocs.push(spec); + } + + peek(): Token { + return this.peekAt(0); + } + + /** Look `n` tokens ahead (0 = the next token). Tokens are scanned + * on demand and buffered, so lookahead never re-scans. */ + peekAt(n: number): Token { + while (this.lookahead.length <= n) this.lookahead.push(this.scanToken()); + return this.lookahead[n]!; + } + + next(): Token { + if (this.lookahead.length > 0) return this.lookahead.shift()!; + return this.scanToken(); + } + + /** Rewind the stream to `pos`, discarding buffered lookahead. Used when + * the parser consumed a region character-wise (test commands). */ + reposition(pos: number): void { + this.pos = pos; + this.lookahead = []; + } + + text(token: Token): string { + return this.source.slice(token.start, token.end); + } + + private scanToken(): Token { + this.budget.progress(); + this.skipBlanks(); + const start = this.pos; + if (this.pos >= this.end) return this.scanBoundary('eof', start, start); + const ch = this.source[this.pos]!; + if (ch === '\n') { + this.pos++; + return this.scanBoundary('newline', start, this.pos); + } + if (ch === '#') { + let sinceTick = 0; + while (this.pos < this.end && this.source[this.pos] !== '\n') { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + this.pos++; + } + return { type: 'comment', start, end: this.pos, heredocBodies: [] }; + } + if (ch === '<' || ch === '>') { + // <( / >( start a process substitution, which is word material. The + // heredoc operators were already excluded: <<( is << + ( … + if (this.source[this.pos + 1] === '(') return this.scanWord(); + return this.scanOp(REDIRECT_OPERATORS); + } + if (ch === '&' || ch === '|' || ch === ';' || ch === '(' || ch === ')') { + // `((` at token start opens arithmetic (an arithmetic command or a + // c-style for header), which is word material, not two parens. + if (ch === '(' && this.source[this.pos + 1] === '(') return this.scanWord(); + return this.scanOp(CONTROL_OPERATORS); + } + if (ch === '{' || ch === '}' || ch === '[' || ch === ']') { + this.pos++; + return { type: 'word', start, end: this.pos, heredocBodies: [] }; + } + if (ch >= '0' && ch <= '9') { + let i = this.pos; + while (i < this.end && isDigitAt(this.source, i)) i++; + const next = this.source[i]; + if (next === '<' || next === '>') { + this.pos = i; + return { type: 'io_number', start, end: i, heredocBodies: [] }; + } + return this.scanWord(); + } + return this.scanWord(); + } + + /** Produce a newline/eof token, scanning any queued heredoc bodies that + * start right after it. */ + private scanBoundary(type: 'newline' | 'eof', start: number, end: number): Token { + const bodies: HeredocBody[] = []; + while (this.pendingHeredocs.length > 0) { + bodies.push(this.readHeredocBody(this.pendingHeredocs.shift()!)); + } + return { type, start, end, heredocBodies: bodies }; + } + + private skipBlanks(): void { + let sinceTick = 0; + for (;;) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[this.pos]; + if (isBlank(ch)) { + this.pos++; + continue; + } + // Line continuation is whitespace, not word material. + if (ch === '\\' && this.source[this.pos + 1] === '\n') { + this.pos += 2; + continue; + } + return; + } + } + + private scanOp(table: readonly string[]): Token { + const start = this.pos; + for (const op of table) { + if (this.source.startsWith(op, this.pos) && this.pos + op.length <= this.end) { + this.pos += op.length; + return { type: 'op', start, end: this.pos, heredocBodies: [] }; + } + } + // Unreachable for the callers above, but never loop forever. + this.pos++; + return { type: 'op', start, end: this.pos, heredocBodies: [] }; + } + + private scanWord(): Token { + const start = this.pos; + // `((...))` at token start: one word token ending right after the + // balanced close (the parser re-scans the range as arithmetic). + if (this.source[start] === '(' && this.source[start + 1] === '(') { + this.pos = scanBalanced(this.source, this.budget, start, this.end, '(', ')').end; + return { type: 'word', start, end: this.pos, heredocBodies: [] }; + } + let i = this.pos; + let sinceTick = 0; + while (i < this.end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[i]!; + if (isBlank(ch) || ch === '\n') break; + if (ch === '&' || ch === '|' || ch === ';' || ch === '(' || ch === ')') break; + if (ch === '{' || ch === '}' || ch === '[' || ch === ']') break; + if (ch === '<' || ch === '>') { + if (this.source[i + 1] === '(') { + i = scanBalancedStatements(this.source, this.budget, i + 1, this.end).end; + continue; + } + break; + } + if (ch === '\\') { + // A line continuation ends the run (it acts as whitespace); a lone + // trailing backslash at end of range is consumed as word text. + if (this.source[i + 1] === '\n') break; + i += 2; + continue; + } + if (ch === '"') { + i = skipDoubleQuoted(this.source, this.budget, i, this.end); + continue; + } + if (ch === "'") { + i = skipSingleQuoted(this.source, this.budget, i, this.end); + continue; + } + if (ch === '`') { + i = skipBacktick(this.source, this.budget, i, this.end); + continue; + } + if (ch === '$') { + i = skipDollar(this.source, this.budget, i, this.end); + continue; + } + i++; + } + if (i === start) i++; // defensive: never emit a zero-width word token + this.pos = i; + return { type: 'word', start, end: i, heredocBodies: [] }; + } + + /** + * Scan one heredoc body, starting at the current position (right after the + * newline that ended the command line). Matches tree-sitter-bash's layout: + * for `<<-` only the first body line's leading tabs are stripped; the end + * marker is the bare delimiter word and any tabs before it belong to the + * body. + */ + private readHeredocBody(spec: HeredocSpec): HeredocBody { + let bodyStart = this.pos; + if (spec.stripTabs) { + while (bodyStart < this.end && this.source[bodyStart] === '\t') bodyStart++; + } + let lineStart = this.pos; + while (lineStart < this.end) { + this.budget.progress(); + let marker = lineStart; + if (spec.stripTabs) { + while (marker < this.end && this.source[marker] === '\t') marker++; + } + if (spec.delimiter.length > 0 && this.source.startsWith(spec.delimiter, marker)) { + const after = marker + spec.delimiter.length; + if (after >= this.end || this.source[after] === '\n') { + this.pos = after; + return { bodyStart, bodyEnd: marker, endStart: marker, endEnd: after, found: true }; + } + } + const newline = this.source.indexOf('\n', lineStart); + if (newline === -1) break; + lineStart = newline + 1; + } + // Unterminated: the body runs to the end of this lexer's range. + this.pos = this.end; + return { bodyStart, bodyEnd: this.end, endStart: this.end, endEnd: this.end, found: false }; + } +} diff --git a/packages/tree-sitter-bash/src/node.ts b/packages/tree-sitter-bash/src/node.ts new file mode 100644 index 0000000000..09e0001892 --- /dev/null +++ b/packages/tree-sitter-bash/src/node.ts @@ -0,0 +1,121 @@ +// src/node.ts +// +// Syntax tree node model. Named node types correspond one-to-one with +// tree-sitter-bash's named node types, so consumers can write tree queries +// against either implementation. Two deliberate deviations from tree-sitter: +// +// - startIndex / endIndex are UTF-16 code unit offsets (native JS string +// indexing), not tree-sitter's UTF-8 byte offsets. `node.text` therefore +// always equals `source.slice(node.startIndex, node.endIndex)`. +// - `text` is pre-computed when the node is created instead of being +// re-sliced on every access. +// +// Child semantics: `children` contains every child in source order, including +// anonymous (punctuation / keyword token) nodes; `namedChildren` contains only +// the named ones, in the same relative order. Anonymous nodes never appear in +// `namedChildren`. `descendantsOfType` walks named descendants only, matching +// how tree-sitter queries see the tree. + +export interface SyntaxNode { + /** Node type, e.g. 'program', 'command', 'word'. Matches tree-sitter-bash. */ + readonly type: string; + /** Source text covered by this node (UTF-16 slice of the original source). */ + readonly text: string; + /** Start offset in UTF-16 code units, inclusive. */ + readonly startIndex: number; + /** End offset in UTF-16 code units, exclusive. */ + readonly endIndex: number; + /** Whether this is a named node (false for punctuation/keyword tokens). */ + readonly isNamed: boolean; + readonly parent: SyntaxNode | null; + /** All children in source order, named and anonymous. */ + readonly children: readonly SyntaxNode[]; + /** Named children only, in source order. */ + readonly namedChildren: readonly SyntaxNode[]; +} + +export interface NodeInit { + type: string; + source: string; + startIndex: number; + endIndex: number; + isNamed?: boolean; +} + +/** + * Mutable node under construction. The parser builds trees with this class and + * exposes them through the readonly `SyntaxNode` interface; once a node is + * handed out it must be treated as immutable. + */ +export class SyntaxNodeBuilder { + readonly type: string; + readonly text: string; + readonly startIndex: number; + readonly endIndex: number; + readonly isNamed: boolean; + parent: SyntaxNodeBuilder | null = null; + readonly children: SyntaxNodeBuilder[] = []; + readonly namedChildren: SyntaxNodeBuilder[] = []; + + constructor(init: NodeInit) { + if (init.startIndex < 0 || init.endIndex < init.startIndex || init.endIndex > init.source.length) { + throw new RangeError( + `invalid node range [${init.startIndex}, ${init.endIndex}) for source of length ${init.source.length}`, + ); + } + this.type = init.type; + this.startIndex = init.startIndex; + this.endIndex = init.endIndex; + this.isNamed = init.isNamed ?? true; + this.text = init.source.slice(init.startIndex, init.endIndex); + } + + /** Attach a child, wiring its parent pointer. Named children are also added + * to `namedChildren`. Returns the child for chaining. + * + * Children must lie inside the parent's range and be appended in source + * order without overlapping the previous sibling (a zero-width child may + * start exactly where the previous sibling ends). */ + addChild(child: T): T { + if (child.parent !== null) throw new Error(`node '${child.type}' already has a parent`); + if (child.startIndex < this.startIndex || child.endIndex > this.endIndex) { + throw new RangeError( + `child '${child.type}' [${child.startIndex}, ${child.endIndex}) escapes parent '${this.type}' [${this.startIndex}, ${this.endIndex})`, + ); + } + const last = this.children.at(-1); + if (last !== undefined && child.startIndex < last.endIndex) { + throw new RangeError( + `child '${child.type}' [${child.startIndex}, ${child.endIndex}) overlaps sibling '${last.type}' [${last.startIndex}, ${last.endIndex})`, + ); + } + child.parent = this; + this.children.push(child); + if (child.isNamed) this.namedChildren.push(child); + return child; + } +} + +/** Convenience factory for a detached node. */ +export function createNode(init: NodeInit): SyntaxNodeBuilder { + return new SyntaxNodeBuilder(init); +} + +/** + * Pre-order traversal of the named descendants of `root` (not including + * `root` itself), filtered to the given types. With no types, returns every + * named descendant in pre-order. Iterative with an explicit stack so that + * pathologically deep trees cannot overflow the call stack. + */ +export function descendantsOfType(root: SyntaxNode, ...types: string[]): SyntaxNode[] { + const wanted = types.length > 0 ? new Set(types) : null; + const out: SyntaxNode[] = []; + const stack: SyntaxNode[] = []; + for (let i = root.namedChildren.length - 1; i >= 0; i--) stack.push(root.namedChildren[i]!); + while (stack.length > 0) { + const node = stack.pop()!; + if (wanted === null || wanted.has(node.type)) out.push(node); + for (let i = node.namedChildren.length - 1; i >= 0; i--) stack.push(node.namedChildren[i]!); + } + return out; +} diff --git a/packages/tree-sitter-bash/src/parse.ts b/packages/tree-sitter-bash/src/parse.ts new file mode 100644 index 0000000000..1e28a83c9d --- /dev/null +++ b/packages/tree-sitter-bash/src/parse.ts @@ -0,0 +1,46 @@ +// src/parse.ts +// +// Public parse entry point. Runs the hand-written lexer + recursive-descent +// parser under a ParseBudget and returns the materialized syntax tree. +// +// Exit contract: +// - Budget exhaustion (time or node cap) → { ok: false, reason: 'aborted' }. +// `Aborted` never escapes. +// - Malformed input → { ok: true, hasError: true } with ERROR nodes and/or +// partial nodes where the parser recovered. +// - Any other parser-internal exception (a bug, not user input) is caught +// here as a last resort: the caller still gets a usable tree — a program +// root with a single ERROR child spanning the whole source and +// hasError: true — instead of an exception. Chosen over reporting +// 'aborted' because the parse did not hit its budget; downstream +// consumers keep working on a degraded tree and hasError signals the +// failure. (See the catch block below.) + +import { Aborted, ParseBudget } from '#/budget'; +import type { BudgetOptions } from '#/budget'; +import { SyntaxNodeBuilder } from '#/node'; +import type { SyntaxNode } from '#/node'; +import { Parser, materialize } from '#/parser'; + +export type ParseResult = + | { ok: true; rootNode: SyntaxNode; hasError: boolean } + | { ok: false; reason: 'aborted' }; + +export type ParseOptions = BudgetOptions; + +export function parse(source: string, options: ParseOptions = {}): ParseResult { + const budget = new ParseBudget(options); + try { + const parser = new Parser(source, budget); + const root = parser.parseProgram(); + const rootNode = materialize(root, source); + return { ok: true, rootNode, hasError: parser.hasError }; + } catch (error) { + if (error instanceof Aborted) return { ok: false, reason: 'aborted' }; + // Last-resort guard for parser bugs: degrade to an ERROR root instead of + // throwing into the caller (see the file header for why). + const root = new SyntaxNodeBuilder({ type: 'program', source, startIndex: 0, endIndex: source.length }); + root.addChild(new SyntaxNodeBuilder({ type: 'ERROR', source, startIndex: 0, endIndex: source.length })); + return { ok: true, rootNode: root, hasError: true }; + } +} diff --git a/packages/tree-sitter-bash/src/parser.ts b/packages/tree-sitter-bash/src/parser.ts new file mode 100644 index 0000000000..be37427c4b --- /dev/null +++ b/packages/tree-sitter-bash/src/parser.ts @@ -0,0 +1,3787 @@ +// src/parser.ts +// +// Recursive-descent bash parser. One method per tree-sitter-bash grammar +// rule (parseList ↔ list, parsePipeline ↔ pipeline, parseCommand ↔ command, +// …), producing node ranges and child layouts that match the real +// tree-sitter-bash 0.25.0 tree. +// +// Construction strategy: the parser builds lightweight `Frame`s, not +// SyntaxNodeBuilders, because heredoc bodies are scanned only when the line +// ends — a heredoc_redirect node (and every ancestor) gets its final range +// after the fact. Frames are mutable and carry parent pointers so the +// heredoc drain can extend the ancestor chain. `materialize` converts the +// finished frame tree into SyntaxNodeBuilders iteratively (explicit stack, +// safe for pathologically deep trees) once parsing is done. +// +// Error recovery: unterminated constructs (quote, expansion, substitution, +// heredoc, compound commands) keep their partial node and set hasError; +// tokens that cannot start or continue a statement are wrapped in an ERROR +// node and parsing continues. No parser-internal exception is expected to +// escape; parse() still guards against it. +// +// Depth guards, all capped so pathological nesting degrades locally +// (ERROR nodes, hasError) instead of overflowing the call stack: +// - word-level substitutions ($( … ) / `…` / <( … )) recurse via fresh +// sub-Parser instances bounded to their source range — the deepest +// chain per level (~13 frames), capped by MAX_SUBSTITUTION_DEPTH; +// - subshells and compound commands (if/while/for/case/{…}/function +// bodies) recurse within one parser (`scopeDepth`); +// - the parseLiteral ↔ parseString ↔ parseExpansion chain behind ${ … } +// nesting — including the pattern → string → $ cycle — has its own +// counter (`literalDepth`, incremented by both parseLiteral and +// parseExpansion); +// - parenthesized_expression nesting inside arithmetic/test expressions +// uses `exprDepth`. +// The last three are capped by MAX_PARSE_DEPTH; beyond the caps the +// construct degrades (compound keywords fall back to plain commands, +// expressions and substitutions to ERROR nodes). The lexer's own scan +// recursion (scanBalanced ↔ skipDoubleQuoted) is cheaper per level and +// capped separately (MAX_SCAN_DEPTH in lexer.ts). +// +// Expression engine: arithmetic expansions ($((…)), ((…)), $[…]), c-style +// for headers and test commands ([[ … ]] / [ … ]) share a small Pratt +// parser (parseExpression over an ExprState) driven by the precedence table +// in grammar.ts, mirroring grammar.js's PREC levels and the tree shapes they +// produce (e.g. prefix -/+/~/! grab a full tighter-precedence expression: +// $((-x + ~y)) is unary(-, binary(x, +, unary(~, y))) in the reference). + +import type { ParseBudget } from '#/budget'; +import { + DECLARATION_COMMAND_KEYWORDS, + EXPRESSION_OPERATORS, + EXPRESSION_PRECEDENCE, + FILE_REDIRECT_OPERATORS, + RESERVED_WORDS, + SPECIAL_VARIABLE_CHARS, + UNSET_COMMAND_KEYWORDS, +} from '#/grammar'; +import { Lexer, scanBalanced, scanBalancedStatements, skipBacktick, skipDollar, skipDoubleQuoted, skipSingleQuoted } from '#/lexer'; +import type { BalancedScan, HeredocBody, HeredocSpec, Token } from '#/lexer'; +import { SyntaxNodeBuilder } from '#/node'; + +/** Maximum nesting of scopes (subshells, compound commands, expressions, + * ${…} chains). These chains cost only a handful of stack frames per + * level and their guards skip iteratively, so 500 is comfortably safe — + * verified: inputs thousands of levels deep degrade locally instead of + * overflowing. */ +export const MAX_PARSE_DEPTH = 500; + +/** + * Maximum nesting of the $( … ) / `…` / <( … ) sub-parser chain (each + * level spawns a fresh Parser and costs ~13 stack frames through + * parseCommandSubstitution → parseScopedStatements → parseStatementList → + * parseCommand → parseLiteral → parseDollar). Measured overflow on a + * default Node stack: ~380–500 levels depending on environment, BEFORE the + * old 500 cap could fire. 150 keeps a ≥2.5× margin; beyond it the + * substitution degrades to a local ERROR node (hasError) instead of + * taking the whole tree down via a RangeError. + */ +export const MAX_SUBSTITUTION_DEPTH = 150; + +const FILE_REDIRECT_OP_SET: ReadonlySet = new Set(FILE_REDIRECT_OPERATORS); +const DECLARATION_COMMAND_SET: ReadonlySet = new Set(DECLARATION_COMMAND_KEYWORDS); +const UNSET_COMMAND_SET: ReadonlySet = new Set(UNSET_COMMAND_KEYWORDS); +const RESERVED_WORD_SET: ReadonlySet = new Set(RESERVED_WORDS); +const NUMBER_RE = /^-?(0x)?[0-9]+(#[0-9A-Za-z@_]+)?$/; +const ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*\+?=/; +const ASSIGNMENT_SPLIT_RE = /^([A-Za-z_][A-Za-z0-9_]*)(\+?=)/; +const SUBSCRIPT_ASSIGNMENT_RE = /^(\w+)\[([^\]\n]*)\](\+?=)/; +const IDENTIFIER_RE = /^[A-Za-z_]\w*$/; +/** Function names additionally allow `:` (`foo::bar() { … }` — the + * reference accepts any such word as a function name). */ +const FUNCTION_NAME_RE = /^[A-Za-z_][\w:]*$/; +const BRACE_EXPRESSION_RE = /^\{(\d+)\.\.(\d+)\}/; +/** Statement-position `((…))` that tree-sitter-bash parses as a + * test_command: inner text starting with an identifier-ish token + * (letters/digits/`_`/`-`) ending in `++`/`--`. */ +const PAREN_TEST_RE = /^\(\(\s*[A-Za-z_][\w-]*(\+\+|--)/; +const STOP_THEN: ReadonlySet = new Set(['then']); +const STOP_DO: ReadonlySet = new Set(['do']); +const STOP_DONE: ReadonlySet = new Set(['done']); +const STOP_IF_BODY: ReadonlySet = new Set(['elif', 'else', 'fi']); +const STOP_FI: ReadonlySet = new Set(['fi']); +const STOP_CLOSE_BRACE: ReadonlySet = new Set(['}']); +const STOP_ESAC: ReadonlySet = new Set(['esac']); +const CASE_TERMINATION_OPS: ReadonlySet = new Set([';;', ';&', ';;&']); + +/** How often long scan loops tick the budget, in scanned characters. */ +const SCAN_TICK_INTERVAL = 2048; + +// Precedence levels from grammar.js's PREC table that are not in +// EXPRESSION_PRECEDENCE. +const PREC_TERNARY = 2; +const PREC_TEST = 10; +const PREC_UNARY = 11; +const PREC_PREFIX = 17; +const PREC_POSTFIX = 18; + +/** Mutable node under construction; see the file header. */ +export interface Frame { + type: string; + start: number; + end: number; + isNamed: boolean; + parent: Frame | null; + children: Frame[]; +} + +interface PendingHeredoc { + frame: Frame; + spec: HeredocSpec; +} + +interface StatementListOptions { + readonly stopAtParen?: boolean; + readonly stopWords?: ReadonlySet; + readonly stopOps?: ReadonlySet; + /** True when the grammar requires a terminator before the stop keyword + * (if/while/do bodies, `{ … }`): stopping with a pending statement and + * no separator flags an error (`if a; then b fi` is an error in the + * reference too). Case items and subshells do not require one. */ + readonly terminatorRequired?: boolean; +} + +/** Where the expression engine runs: arithmetic expansions, c-style for + * headers, or test commands. */ +type ExprMode = 'arith' | 'c' | 'test'; + +type ExprTokenKind = + | 'number' + | 'ident' + | 'word' + | 'subst' + | 'string' + | 'testop' + | 'op' + | 'lparen' + | 'rparen' + | 'unknown' + | 'end'; + +interface ExprToken { + readonly kind: ExprTokenKind; + readonly start: number; + readonly end: number; + readonly text: string; + /** Pre-built node for subst/string/subscript tokens. */ + readonly frame?: Frame; +} + +interface ExprState { + pos: number; + readonly end: number; + readonly mode: ExprMode; + lookahead: ExprToken | null; + /** Nesting of parenthesized_expression inside test commands (the + * reference emits word, not extglob_pattern, for a non-glob ==/!= + * right-hand side inside parentheses). */ + parenDepth: number; + /** Test mode only: true after a complete operand, where `=`/`!`-family + * characters are comparison operators even when attached to the next + * word (`a ==b` is `a == b`); false at operand position, where they are + * word characters (`=b`, `!x`, `a!=b` are words in the reference). */ + expectOperator: boolean; +} + +function isFileRedirectOp(text: string): boolean { + return FILE_REDIRECT_OP_SET.has(text); +} + +/** Strip quoting from a heredoc delimiter word; any quote or backslash in + * the raw text marks the body as quoted (no expansions). */ +function extractHeredocSpec(raw: string, stripTabs: boolean): HeredocSpec { + let delimiter = ''; + let quoted = false; + for (let i = 0; i < raw.length; i++) { + const ch = raw[i]!; + if (ch === '\\' && i + 1 < raw.length) { + delimiter += raw[i + 1]; + quoted = true; + i++; + } else if (ch === '"' || ch === "'") { + quoted = true; + } else { + delimiter += ch; + } + } + return { delimiter, stripTabs, quoted }; +} + +export class Parser { + /** Set when any recovery path was taken. */ + hasError = false; + private lexer!: Lexer; + private readonly heredocQueue: PendingHeredoc[] = []; + private scopeDepth = 0; + /** Recursion depth of parseLiteral ↔ parseString / parseExpansion (the + * ${} nesting chain that stays inside this Parser instance). */ + private literalDepth = 0; + /** Nesting of parenthesized_expression inside the expression parser. */ + private exprDepth = 0; + /** True while parsing inside a heredoc's line tail: a second heredoc + * cannot be represented and degrades to ERROR (see parseHeredocRedirect). */ + private noHeredoc = false; + /** Nesting of case_item statement lists (see parseCaseItemStatements). */ + private caseItemDepth = 0; + /** End of the most recently consumed `;`/`&`/`;;`/newline terminator in + * any statement list. case_item uses it to extend its range over a + * trailing terminator before `esac`, matching the reference (whose + * last_case_item includes the final _terminator in its range). */ + private lastTerminatorEnd = 0; + + constructor( + private readonly source: string, + private readonly budget: ParseBudget, + private readonly depth = 0, + ) {} + + // ---------------------------------------------------------------- helpers + + private frame(type: string, start: number, end: number, children: Frame[] = [], isNamed = true): Frame { + this.budget.tick(); + const frame: Frame = { type, start, end, isNamed, parent: null, children }; + for (const child of children) child.parent = frame; + return frame; + } + + private anon(type: string, start: number, end: number): Frame { + return this.frame(type, start, end, [], false); + } + + private addKid(parent: Frame, child: Frame): void { + child.parent = parent; + parent.children.push(child); + } + + private text(start: number, end: number): string { + return this.source.slice(start, end); + } + + private tokenText(token: Token): string { + return this.source.slice(token.start, token.end); + } + + private endOf(kids: Frame[], fallback: number): number { + return kids.length > 0 ? kids.at(-1)!.end : fallback; + } + + private isStatementStart(token: Token): boolean { + if (token.type === 'word' || token.type === 'io_number') return true; + if (token.type !== 'op') return false; + const text = this.tokenText(token); + return text === '(' || text === '<<<' || text === '<<' || text === '<<-' || isFileRedirectOp(text); + } + + /** Peeked word token whose text equals `keyword`. */ + private peekKeyword(keyword: string): boolean { + const token = this.lexer.peek(); + return token.type === 'word' && this.tokenText(token) === keyword; + } + + /** Consume a keyword as an anonymous child; false (no consume) when the + * next token is not that keyword. */ + private consumeKeyword(kids: Frame[], keyword: string): boolean { + if (!this.peekKeyword(keyword)) return false; + const token = this.lexer.next(); + kids.push(this.anon(keyword, token.start, token.end)); + return true; + } + + // ------------------------------------------------------------ entry point + + /** program: the whole source as one statement list. */ + parseProgram(): Frame { + this.lexer = new Lexer(this.source, this.budget, 0, this.source.length); + const children = this.parseStatementList(); + return this.frame('program', 0, this.source.length, children); + } + + /** Parse a sub-range as a statement list (body of $( … ), ` … `, <( … )). + * The sub-parser chain is the deepest per-level recursion in the + * parser, so it has its own, tighter cap — see MAX_SUBSTITUTION_DEPTH. */ + private parseScopedStatements(start: number, end: number): Frame[] { + if (this.depth + 1 >= MAX_SUBSTITUTION_DEPTH) { + this.hasError = true; + return [this.frame('ERROR', start, end)]; + } + const sub = new Parser(this.source, this.budget, this.depth + 1); + sub.lexer = new Lexer(this.source, this.budget, start, end); + const children = sub.parseStatementList(); + if (sub.hasError) this.hasError = true; + return children; + } + + // -------------------------------------------------------- statement lists + + /** + * _statements: statements separated/terminated by ; & and newlines. + * `;`/`&`/`;;` terminators become anonymous children; newlines and + * comments do not (comments stay named children) — tree-sitter-bash never + * emits `\n` terminator nodes either. Parsing stops (without consuming) + * at a word in stopWords (then/do/done/fi/esac/…), an op in stopOps + * (case_item terminators), or a `)` when stopAtParen is set. + */ + private parseStatementList(options: StatementListOptions = {}): Frame[] { + const children: Frame[] = []; + let needTerminator = false; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'eof') { + this.completeHeredocs(token.heredocBodies); + this.failOpenHeredocs(); + break; + } + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + this.lastTerminatorEnd = token.end; + needTerminator = false; + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + children.push(this.frame('comment', token.start, token.end)); + continue; + } + if (token.type === 'word' && options.stopWords?.has(this.tokenText(token)) === true) { + if (needTerminator && options.terminatorRequired === true) this.hasError = true; // missing separator + break; + } + const op = token.type === 'op' ? this.tokenText(token) : ''; + if (token.type === 'op' && options.stopOps?.has(op) === true) break; + if (token.type === 'op' && op === ')' && options.stopAtParen === true) { + this.failOpenHeredocs(); + break; + } + if (token.type === 'op' && (op === ';' || op === '&' || op === ';;')) { + this.lexer.next(); + children.push(this.anon(op, token.start, token.end)); + this.lastTerminatorEnd = token.end; + needTerminator = false; + continue; + } + if ( + token.type === 'op' && + (op === ')' || op === '&&' || op === '||' || op === '|' || op === '|&' || op === ';&' || op === ';;&') + ) { + // A binary operator or closer with no left-hand side: recover. + this.hasError = true; + this.lexer.next(); + children.push(this.frame('ERROR', token.start, token.end, [this.anon(op, token.start, token.end)])); + needTerminator = false; + continue; + } + if (needTerminator) this.hasError = true; // missing separator, e.g. `cmd (sub)` + children.push(this.parseList()); + needTerminator = true; + } + return children; + } + + /** Consume newlines and comments in operator-continuation position + * (`a &&\n b`). Returns the comment nodes; newlines are dropped. */ + private skipContinuation(): Frame[] { + const comments: Frame[] = []; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + comments.push(this.frame('comment', token.start, token.end)); + continue; + } + return comments; + } + } + + // ------------------------------------------------------------- statements + + /** list: statement ((&& | ||) statement)* — left associative. */ + private parseList(): Frame { + let left = this.parsePipeline(); + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'op') break; + const op = this.tokenText(token); + if (op !== '&&' && op !== '||') break; + this.lexer.next(); + const extras = this.skipContinuation(); + if (this.isStatementStart(this.lexer.peek())) { + const right = this.parsePipeline(); + left = this.frame('list', left.start, right.end, [left, this.anon(op, token.start, token.end), ...extras, right]); + } else { + // Trailing connector (`ls &&`): keep the partial list, flag it. + this.hasError = true; + left = this.frame('list', left.start, token.end, [left, this.anon(op, token.start, token.end)]); + break; + } + } + return left; + } + + /** pipeline: statement ((| | |&) statement)* */ + private parsePipeline(): Frame { + return this.parsePipelineTail(this.parseStatementNotPipeline()); + } + + private parsePipelineTail(first: Frame): Frame { + const kids: Frame[] = [first]; + let end = first.end; + let pipes = 0; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'op') break; + const op = this.tokenText(token); + if (op !== '|' && op !== '|&') break; + this.lexer.next(); + pipes++; + kids.push(this.anon(op, token.start, token.end)); + end = token.end; + kids.push(...this.skipContinuation()); + if (this.isStatementStart(this.lexer.peek())) { + const next = this.parseStatementNotPipeline(); + kids.push(next); + end = next.end; + } else { + this.hasError = true; // dangling pipe (`ls |`) + break; + } + } + if (pipes === 0) return first; + return this.frame('pipeline', first.start, end, kids); + } + + /** _statement_not_pipeline: any statement form plus trailing redirects. */ + private parseStatementNotPipeline(): Frame { + let inner: Frame | null = this.parseStatementCore(); + const trailing: Frame[] = []; + for (;;) { + const next = this.lexer.peek(); + if (next.type === 'io_number') { + trailing.push(this.parseRedirect()); + continue; + } + if (next.type === 'op') { + const op = this.tokenText(next); + if (isFileRedirectOp(op) || op === '<<<' || op === '<<' || op === '<<-') { + trailing.push(this.parseRedirect()); + continue; + } + } + break; + } + if (trailing.length > 0) { + const kids = inner === null ? trailing : [inner, ...trailing]; + inner = this.frame('redirected_statement', kids[0]!.start, kids.at(-1)!.end, kids); + } + if (inner === null) { + // Unreachable from well-formed dispatch; recover without looping. + this.hasError = true; + const skipped = this.lexer.next(); + inner = this.frame('ERROR', skipped.start, skipped.end); + } + return inner; + } + + /** + * Dispatch one statement (no pipeline, no trailing redirects) on its + * leading token: subshell, negation, compound commands, test commands, + * declaration/unset commands, function definitions, or a plain command. + * Reserved words only count at statement position — `echo if` keeps `if` + * as a word, and a prefix assignment (`x=1 if …`) disables the keyword + * reading just like in the reference. + */ + private parseStatementCore(): Frame | null { + const token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '(') { + return this.parseSubshell(); + } + if (token.type === 'word') { + const text = this.tokenText(token); + if (text === '!') return this.parseNegatedCommand(); + if (text === '{') { + // `{1..3}` at statement position is a brace_expression word. + if (!BRACE_EXPRESSION_RE.test(this.source.slice(token.start))) { + return this.parseCompoundGuarded(() => this.parseCompoundStatement()); + } + } else if (text === 'if') { + return this.parseCompoundGuarded(() => this.parseIfStatement()); + } else if (text === 'while' || text === 'until') { + return this.parseCompoundGuarded(() => this.parseWhileStatement()); + } else if (text === 'for' || text === 'select') { + return this.parseCompoundGuarded(() => this.parseForStatement()); + } else if (text === 'case') { + return this.parseCompoundGuarded(() => this.parseCaseStatement()); + } else if (text === 'function') { + return this.parseCompoundGuarded(() => this.parseFunctionDefinition()); + } else if (text === '[') { + return this.parseTestCommand(); + } else if (text.startsWith('((') && PAREN_TEST_RE.test(text)) { + // `((word++…))` / `((word--…))` at statement position is a + // test_command in the reference (its statement-position arithmetic + // grammar does not accept a leading postfix operand), while other + // `((…))` forms are arithmetic_expansion command names. + return this.parseParenTestCommand(); + } else if (DECLARATION_COMMAND_SET.has(text)) { + return this.parseDeclarationCommand(); + } else if (UNSET_COMMAND_SET.has(text)) { + return this.parseUnsetCommand(); + } else if (this.isFunctionDefinitionAhead()) { + return this.parseCompoundGuarded(() => this.parseFunctionDefinition()); + } + } + return this.parseCommand(); + } + + /** + * Run a compound-statement parser under the scope-depth guard: beyond + * MAX_PARSE_DEPTH the keyword is parsed as a plain command word instead, + * which bounds the recursion chain (if → statements → if → …). + */ + private parseCompoundGuarded(parse: () => Frame): Frame { + if (this.scopeDepth >= MAX_PARSE_DEPTH) { + this.hasError = true; + const fallback = this.parseCommand(); + if (fallback !== null) return fallback; + const token = this.lexer.next(); + return this.frame('ERROR', token.start, token.end); + } + this.scopeDepth++; + try { + return parse(); + } finally { + this.scopeDepth--; + } + } + + /** negated_command: `!` followed by a command, test command, assignment + * or subshell (the grammar's full choice). */ + private parseNegatedCommand(): Frame { + const bang = this.lexer.next(); + const kids: Frame[] = [this.anon('!', bang.start, bang.end)]; + let end = bang.end; + const token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '(') { + const subshell = this.parseSubshell(); + kids.push(subshell); + end = subshell.end; + } else if (token.type === 'word' && this.tokenText(token) === '[') { + const test = this.parseTestCommand(); + kids.push(test); + end = test.end; + } else if (token.type === 'word' && this.tokenText(token).startsWith('((') && PAREN_TEST_RE.test(this.tokenText(token))) { + const test = this.parseParenTestCommand(); + kids.push(test); + end = test.end; + } else { + const command = this.parseCommand(); + if (command === null) { + this.hasError = true; + } else { + kids.push(command); + end = command.end; + } + } + return this.frame('negated_command', bang.start, end, kids); + } + + /** subshell: `(` _statements `)` */ + private parseSubshell(): Frame { + const open = this.lexer.next(); + if (this.scopeDepth >= MAX_PARSE_DEPTH) { + // Absurd nesting: skip token-wise to the matching close paren. Word + // tokens hide their internal parens, so token-level counting is exact. + this.hasError = true; + let depth = 1; + let end = open.end; + for (;;) { + const token = this.lexer.next(); + end = token.end; + if (token.type === 'eof') break; + this.completeHeredocs(token.heredocBodies); + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === '(') depth++; + else if (op === ')') { + depth--; + if (depth === 0) break; + } + } + } + return this.frame('ERROR', open.start, end, [this.anon('(', open.start, open.end)]); + } + const kids: Frame[] = [this.anon('(', open.start, open.end)]; + this.scopeDepth++; + const inner = this.parseStatementList({ stopAtParen: true }); + this.scopeDepth--; + if (inner.length === 0) this.hasError = true; // empty subshell (grammar requires statements) + kids.push(...inner); + let end = this.endOf(inner, open.end); + const token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === ')') { + this.lexer.next(); + kids.push(this.anon(')', token.start, token.end)); + end = token.end; + } else { + this.hasError = true; // unterminated subshell + } + return this.frame('subshell', open.start, end, kids); + } + + // ------------------------------------------------------ compound commands + + /** compound_statement: `{` _terminated_statement `}` */ + private parseCompoundStatement(): Frame { + const open = this.lexer.next(); + const kids: Frame[] = [ + this.anon('{', open.start, open.end), + ...this.parseStatementList({ stopWords: STOP_CLOSE_BRACE, terminatorRequired: true }), + ]; + let end = this.endOf(kids, open.end); + const token = this.lexer.peek(); + if (token.type === 'word' && this.tokenText(token) === '}') { + this.lexer.next(); + kids.push(this.anon('}', token.start, token.end)); + end = token.end; + } else { + this.hasError = true; // unterminated { … + } + return this.frame('compound_statement', open.start, end, kids); + } + + /** if_statement: `if` condition `then` body elif* else? `fi` */ + private parseIfStatement(): Frame { + const ifToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('if', ifToken.start, ifToken.end), + ...this.parseStatementList({ stopWords: STOP_THEN, terminatorRequired: true }), + ]; + if (!this.consumeKeyword(kids, 'then')) { + this.hasError = true; // missing then + return this.frame('if_statement', ifToken.start, this.endOf(kids, ifToken.end), kids); + } + kids.push(...this.parseStatementList({ stopWords: STOP_IF_BODY, terminatorRequired: true })); + while (this.peekKeyword('elif')) { + kids.push(this.parseElifClause()); + } + if (this.peekKeyword('else')) { + kids.push(this.parseElseClause()); + } + let end = this.endOf(kids, ifToken.end); + if (this.consumeKeyword(kids, 'fi')) { + end = kids.at(-1)!.end; + } else { + this.hasError = true; // unterminated if + } + return this.frame('if_statement', ifToken.start, end, kids); + } + + /** elif_clause: `elif` condition `then` body */ + private parseElifClause(): Frame { + const elifToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('elif', elifToken.start, elifToken.end), + ...this.parseStatementList({ stopWords: STOP_THEN, terminatorRequired: true }), + ]; + if (!this.consumeKeyword(kids, 'then')) { + this.hasError = true; + return this.frame('elif_clause', elifToken.start, this.endOf(kids, elifToken.end), kids); + } + kids.push(...this.parseStatementList({ stopWords: STOP_IF_BODY, terminatorRequired: true })); + // Like case_item, an elif/else clause extends over its trailing + // terminator (usually a newline) in the reference. + const end = Math.max(this.endOf(kids, elifToken.end), this.lastTerminatorEnd); + return this.frame('elif_clause', elifToken.start, end, kids); + } + + /** else_clause: `else` body */ + private parseElseClause(): Frame { + const elseToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('else', elseToken.start, elseToken.end), + ...this.parseStatementList({ stopWords: STOP_FI, terminatorRequired: true }), + ]; + const end = Math.max(this.endOf(kids, elseToken.end), this.lastTerminatorEnd); + return this.frame('else_clause', elseToken.start, end, kids); + } + + /** while_statement: (`while` | `until`) condition do_group */ + private parseWhileStatement(): Frame { + const kwToken = this.lexer.next(); + const keyword = this.tokenText(kwToken); + const kids: Frame[] = [ + this.anon(keyword, kwToken.start, kwToken.end), + ...this.parseStatementList({ stopWords: STOP_DO, terminatorRequired: true }), + ]; + if (this.peekKeyword('do')) { + kids.push(this.parseDoGroup()); + } else { + this.hasError = true; // missing do + } + return this.frame('while_statement', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + /** do_group: `do` body `done` */ + private parseDoGroup(): Frame { + const doToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('do', doToken.start, doToken.end), + ...this.parseStatementList({ stopWords: STOP_DONE, terminatorRequired: true }), + ]; + let end = this.endOf(kids, doToken.end); + if (this.consumeKeyword(kids, 'done')) { + end = kids.at(-1)!.end; + } else { + this.hasError = true; // unterminated do + } + return this.frame('do_group', doToken.start, end, kids); + } + + /** Consume the terminator between a for-header and its body: `;`/`&` + * become anonymous children, newlines are dropped, comments are kept. */ + private consumeForTerminator(kids: Frame[]): void { + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === ';' || op === '&') { + this.lexer.next(); + kids.push(this.anon(op, token.start, token.end)); + } + } + break; + } + } + + /** for_statement: (`for` | `select`) variable (`in` values)? terminator + * do_group — or, for `for ((…))`, a c_style_for_statement. */ + private parseForStatement(): Frame { + const kwToken = this.lexer.next(); + const keyword = this.tokenText(kwToken); + const kids: Frame[] = [this.anon(keyword, kwToken.start, kwToken.end)]; + const next = this.lexer.peek(); + if (keyword === 'for' && next.type === 'word' && this.tokenText(next).startsWith('((')) { + return this.parseCStyleForStatement(kwToken, kids); + } + const varToken = this.lexer.peek(); + if (varToken.type === 'word' && /^\w+$/.test(this.tokenText(varToken))) { + this.lexer.next(); + kids.push(this.frame('variable_name', varToken.start, varToken.end)); + } else { + this.hasError = true; // missing loop variable + } + if (this.peekKeyword('in')) { + const inToken = this.lexer.next(); + const values: Frame[] = []; + while (this.lexer.peek().type === 'word') { + values.push(this.parseWordArgument()); + } + if (values.length === 0) { + // `for x in; do` — the reference wraps the bare `in` in ERROR. + this.hasError = true; + kids.push(this.frame('ERROR', inToken.start, inToken.end, [this.anon('in', inToken.start, inToken.end)])); + } else { + kids.push(this.anon('in', inToken.start, inToken.end)); + kids.push(...values); + } + } + this.consumeForTerminator(kids); + if (this.peekKeyword('do')) { + kids.push(this.parseDoGroup()); + } else { + this.hasError = true; // missing do + } + return this.frame('for_statement', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + /** c_style_for_statement: `for` `((` init `;` cond `;` update `))` + * terminator (do_group | compound_statement). The whole ((…)) header + * arrived as one word token (see the lexer). */ + private parseCStyleForStatement(kwToken: Token, kids: Frame[]): Frame { + const header = this.lexer.next(); + const closed = + header.end - header.start >= 4 && this.source[header.end - 2] === ')' && this.source[header.end - 1] === ')'; + if (!closed) this.hasError = true; + kids.push(this.anon('((', header.start, header.start + 2)); + const innerEnd = closed ? header.end - 2 : header.end; + kids.push(...this.parseCForBody(header.start + 2, innerEnd)); + if (closed) { + kids.push(this.anon('))', header.end - 2, header.end)); + } + this.consumeForTerminator(kids); + if (this.peekKeyword('do')) { + kids.push(this.parseDoGroup()); + } else if (this.peekKeyword('{')) { + kids.push(this.parseCompoundStatement()); + } else { + this.hasError = true; // missing body + } + return this.frame('c_style_for_statement', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + /** The init/condition/update parts of a c-style for header: three + * comma-separated c-expression lists divided by `;`. */ + private parseCForBody(start: number, end: number): Frame[] { + const st = this.newExprState(start, end, 'c'); + const kids: Frame[] = []; + for (let part = 0; part < 3; part++) { + for (;;) { + const expression = this.parseExpression(st, 0); + if (expression !== null) kids.push(expression); + const token = this.exprPeek(st); + if (token.kind === 'op' && token.text === ',') { + this.exprNext(st); + kids.push(this.anon(',', token.start, token.end)); + continue; + } + break; + } + if (part < 2) { + const token = this.exprPeek(st); + if (token.kind === 'op' && token.text === ';') { + this.exprNext(st); + kids.push(this.anon(';', token.start, token.end)); + } else { + this.hasError = true; // missing ; in for-header + break; + } + } + } + const leftover = this.exprLeftover(st); + if (leftover !== null) kids.push(leftover); + return kids; + } + + /** case_statement: `case` value `in` case_item* `esac` */ + private parseCaseStatement(): Frame { + const caseToken = this.lexer.next(); + const kids: Frame[] = [this.anon('case', caseToken.start, caseToken.end)]; + if (this.lexer.peek().type === 'word') { + kids.push(this.parseWordArgument()); + } else { + this.hasError = true; // missing value + } + this.skipCaseTerminators(kids); + if (!this.consumeKeyword(kids, 'in')) { + this.hasError = true; + return this.frame('case_statement', caseToken.start, this.endOf(kids, caseToken.end), kids); + } + this.skipCaseTerminators(kids); + for (;;) { + // Between items: newlines are dropped, comments kept as children. + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + break; + } + const token = this.lexer.peek(); + if (token.type === 'eof') { + this.hasError = true; // missing esac + break; + } + if (token.type === 'word' && this.tokenText(token) === 'esac') break; + const before = this.lexer.pos; + kids.push(this.parseCaseItem()); + if (this.lexer.pos === before) { + // Defensive: a case_item that consumed nothing would loop forever. + this.hasError = true; + this.lexer.next(); + } + if (this.peekKeyword('esac')) { + // A fallthrough terminator on the LAST item is a grammar error in + // the reference (last_case_item only allows `;;`). + const last = kids.at(-1)!.children.at(-1); + if (last !== undefined && !last.isNamed && (last.type === ';&' || last.type === ';;&')) { + this.hasError = true; + } + } + } + let end = this.endOf(kids, caseToken.end); + if (this.consumeKeyword(kids, 'esac')) { + end = kids.at(-1)!.end; + } + return this.frame('case_statement', caseToken.start, end, kids); + } + + /** Terminators around `in` / between case items: newlines dropped, `;`/`&` + * kept as anonymous children, comments kept. */ + private skipCaseTerminators(kids: Frame[]): void { + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === ';' || op === '&') { + this.lexer.next(); + kids.push(this.anon(op, token.start, token.end)); + continue; + } + } + break; + } + } + + /** + * case_item: `(`? pattern (`|` pattern)* `)` statements? terminator? + * Patterns are literals, except bare words that contain a glob character + * (* ? [) or that are followed by `|` — those are extglob_pattern nodes, + * matching the reference scanner. + */ + /** case_item: `( pattern (| pattern)* ) statements? terminator?`. + * Patterns are scanned character-wise (they may hold extglob groups with + * parens the token stream cannot represent) and classified following the + * reference scanner: a bare literal is a word (or concatenation pieces), + * glob material is one extglob_pattern, and a pattern mixing glob text + * with a quote/expansion splits into extglob_pattern pieces around the + * construct (the grammar's _extglob_blob). */ + private parseCaseItem(): Frame { + const kids: Frame[] = []; + let token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '(') { + this.lexer.next(); + kids.push(this.anon('(', token.start, token.end)); + } + // Patterns. + let firstAlternative = true; + for (;;) { + token = this.lexer.peek(); + if (token.type !== 'word') { + this.hasError = true; // missing pattern + break; + } + const alt = this.scanCasePatternEnd(token.start); + kids.push(...this.parseCasePattern(token.start, alt.end, firstAlternative)); + firstAlternative = false; + this.lexer.reposition(alt.end); + token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '|') { + this.lexer.next(); + kids.push(this.anon('|', token.start, token.end)); + continue; + } + break; + } + token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === ')') { + this.lexer.next(); + kids.push(this.anon(')', token.start, token.end)); + } else { + this.hasError = true; // missing ) + } + kids.push(...this.parseCaseItemStatements()); + token = this.lexer.peek(); + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === ';;' || op === ';&' || op === ';;&') { + this.lexer.next(); + kids.push(this.anon(op, token.start, token.end)); + } + } + const start = kids.length > 0 ? kids[0]!.start : token.start; + // An item closed by `esac` (no `;;`) extends over its trailing + // terminator (usually a newline) in the reference. + const end = Math.max(this.endOf(kids, token.start), this.lastTerminatorEnd); + return this.frame('case_item', start, end, kids); + } + + /** End of one case pattern alternative: the position of the top-level `)` + * or `|` (extglob group parens nest). Quote/escape/substitution aware. */ + private scanCasePatternEnd(i: number): { end: number } { + const end = this.lexer.rangeEnd; + let depth = 0; + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\n' || ch === ';') break; // defensive: malformed item + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (ch === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if (ch === '(') { + depth++; + j++; + continue; + } + if (ch === ')') { + if (depth === 0) return { end: j }; + depth--; + j++; + continue; + } + if (ch === '|' && depth === 0) return { end: j }; + if ((ch === ' ' || ch === '\t' || ch === '\r') && depth === 0) return { end: j }; + j++; + } + return { end: j }; + } + + /** Build the frames for one case pattern alternative. The extglob_blob + * piece split (glob text around a quote/expansion) only applies to the + * FIRST alternative — after `|` the reference keeps such patterns as + * plain concatenations. */ + private parseCasePattern(start: number, end: number, allowBlob: boolean): Frame[] { + const raw = this.text(start, end); + // Extglob group present: accepted groups make the whole alternative one + // extglob_pattern; rejected ones degrade with hasError (the reference + // reparses the group as a new item — a recovery shape we do not mirror). + // `.(` is not a sigil, but the reference rejects it the same way. + const group = /[?*+@!.]\(/.exec(raw); + if (group !== null) { + if (this.extglobGroupAccepted(start, end)) return [this.frame('extglob_pattern', start, end)]; + this.hasError = true; + return [this.frame('ERROR', start, end)]; + } + if (!/["'$`]/.test(raw)) { + // Bare pattern. + if (this.isBareGlobPattern(start, end)) return [this.frame('extglob_pattern', start, end)]; + return [this.parseLiteral(start, end)]; + } + // Glob text mixed with a quote/expansion: extglob_pattern pieces around + // a single construct when the leading run qualifies as a glob. + if (allowBlob) { + const pieces = this.parseExtglobBlob(start, end); + if (pieces !== null) return pieces; + } + return [this.parseLiteral(start, end)]; + } + + /** + * Whether the extglob scanner accepts a group-containing pattern + * (scanner.c's first/second character rules): the first character must be + * a letter or one of `?*+@!-)\.[` (a leading escape consumes the escaped + * space/quote with it, plus one more character), and the character at the + * scanner's check position must be alphanumeric or one of `([?/\_*`. + */ + private extglobGroupAccepted(start: number, end: number): boolean { + const isAlpha = (ch: string): boolean => /[A-Za-z]/.test(ch); + const isAlnum = (ch: string): boolean => /[A-Za-z0-9]/.test(ch); + const c0 = this.source[start]!; + if (!isAlpha(c0) && !'?*+@!-)\\.['.includes(c0)) return false; + if (c0 === '[') return true; // the scanner skips the second-char check + let check = start + 1; + if (c0 === '\\') { + const after = this.source[start + 1]; + if (after === undefined || !(after === ' ' || after === '\t' || after === '\r' || after === '"')) return false; + check = start + 3; // escape pair + one unconditionally consumed char + } + if (check >= end) return false; + const ch = this.source[check]!; + return isAlnum(ch) || '([?/\\_*'.includes(ch); + } + + /** + * Whether a bare case pattern (no quotes, substitutions or groups) is an + * extglob_pattern in the reference, derived from scanner.c's + * extglob_pattern scanner (the same source as isTestExtglob for `[[ ]]` + * right sides) and verified against the wasm build on an ~90-pattern + * matrix (foo_bar/word1/abc1d/v2.0/a1/a_ → glob; abc/a.b/1a/é/a-b → word): + * 1. the first character must be a (Unicode) letter or one of + * `?*+@!-)\.[`, else the pattern is a literal; + * 2. a leading single dash makes the pattern a word when the rest is + * plain alphanumerics (the scanner's special-variable branch eats + * the `-` before the glob scan: `-o`, `-xy`, `-abc` → word, `-1` → + * number); `--anything` is a glob, and so is a single dash with an + * inner dash run (`-x-`, `-x-y`, `-x-y-z` → glob) unless the inner + * dashes are doubled (`-x--` → word); + * 3. single-character patterns use the scanner's early exits: a letter + * before `)` is a word, anything else (and anything before `|` or + * whitespace) is a glob; + * 4. a dash in second position is consumed together with its + * alphanumeric run BEFORE the validity check; the pattern is a word + * when the run ends the pattern directly before `)` (`a-b)`, `z-)`) + * or is followed by `.` / a character that is not valid pattern + * material (`a-b-c`). Before `|` or whitespace the glob survives + * (`a-b|`, `a-b `); + * 5. the second character must be alphanumeric or one of `([?/\_*` + * (`*.txt`, `a.b`, `q+w`, `a:b` → word; skipped for a leading `[`); + * 6. ending quirk: a pattern directly followed by `|` is always a glob; + * 7. otherwise the pattern is a glob iff some character counts as glob + * material: the first character when it is not a letter, or any later + * character that is neither a letter nor `.` (digits and `_` count — + * `foo_bar`, `abc1d` are globs; `abc`, `foo.txt` are words). + */ + private isBareGlobPattern(start: number, end: number): boolean { + const raw = this.text(start, end); + if (raw.includes('\\')) return false; + if (NUMBER_RE.test(raw)) return false; // 12, -1, 0x1F → number + const isAlpha = (ch: string): boolean => /^\p{L}$/u.test(ch); + const isAlnum = (ch: string): boolean => /^[\p{L}\p{N}]$/u.test(ch); + const next = this.source[end] ?? ''; + const c0 = raw[0]!; + // (1) first character + if (!isAlpha(c0) && !'?*+@!-)\\.['.includes(c0)) return false; + // (2) leading dash: the scanner's special-variable branch eats a + // leading `-` before the glob scan. A single dash followed by plain + // alphanumerics is a word/number (`-o`, `-xy`, `-x1` → word, `-1` → + // number); `--anything` is a glob; so is a single dash with exactly + // one INNER dash run (`-x-`, `-x-y`, `-x-y-z` → glob) — but a doubled + // inner dash falls back to a word (`-x--`). + if (c0 === '-') { + if (raw.length > 1 && raw[1] === '-') return true; + let innerDash = -1; + for (let i = 1; i < raw.length; i++) { + if (raw[i] === '-') { + innerDash = i; + break; + } + } + if (innerDash === -1) return false; + return raw[innerDash + 1] !== '-'; + } + // (3) single-character patterns: the scanner's early exits — but only + // at a real pattern END (`)`, `|` or whitespace; when a quote or + // substitution follows, the run continues as a blob piece and the + // glob-material check below decides). + if (raw.length === 1) { + if (next === ')') return !isAlpha(c0); + if (next === '|' || next === ' ' || next === '\t' || next === '\r' || next === '\n' || next === '') return true; + } + // (4) a dash in second position is consumed together with its + // alphanumeric run BEFORE the validity check; the pattern is a word + // when the run ends the pattern directly before `)` (a-b), z-)) or is + // followed by `.` / a character that is not valid pattern material + // (a-b-c). Before `|` or whitespace the glob survives (a-b|, a-b␣). + if (raw.length > 1) { + let p = 1; + if (raw[1] === '-') { + let i = 2; + while (i < raw.length && isAlnum(raw[i]!)) i++; + if (i >= raw.length) { + if (next === ')') return false; + return true; + } + const after = raw[i]!; + if (after === '.' || after === '\\') return false; + if (!isAlnum(after) && !'([?/\\_*'.includes(after)) return false; + p = i; + } + // (5) second-character validity (skipped for a leading `[`) + if (c0 !== '[' && p === 1 && !isAlnum(raw[1]!) && !'([?/\\_*'.includes(raw[1]!)) return false; + } + // (6) ending quirk: directly before `|` the scanner accepts unconditionally + if (next === '|') return true; + // (7) glob material? + if (!isAlpha(c0)) return true; + for (let i = 1; i < raw.length; i++) { + const ch = raw[i]!; + if (ch !== '.' && !isAlpha(ch)) return true; + } + return false; + } + + /** + * The _extglob_blob shape: extglob_pattern pieces around ONE construct + * (string / raw_string / expansion / command_substitution). Returns null + * when the leading bare run does not qualify as a glob (the pattern is a + * plain concatenation in the reference) or the shape does not fit. + */ + private parseExtglobBlob(start: number, end: number, leadValidator?: (s: number, e: number) => boolean): Frame[] | null { + // Find the first construct. + let i = start; + while (i < end) { + const ch = this.source[i]!; + if (ch === '\\') { + i += 2; + continue; + } + if (ch === '"' || ch === "'" || ch === '`' || ch === '$') break; + i++; + } + if (i >= end || i === start) return null; + // The leading bare run must qualify as a glob. Runs with escapes count + // when they contain an explicit glob character (`*\ ${x}` → extglob + // "*\ "); unescaped runs go through the bare-pattern rule. A custom + // validator applies for group-containing runs (test patterns). + const leadText = this.text(start, i); + const leadIsGlob = + leadValidator !== undefined + ? leadValidator(start, i) + : leadText.includes('\\') + ? /[*?[]/.test(leadText) + : this.isBareGlobPattern(start, i); + if (!leadIsGlob) return null; + const lead = this.frame('extglob_pattern', start, i); + // The construct. + let construct: Frame; + let next: number; + const ch = this.source[i]!; + if (ch === '"') { + [construct, next] = this.parseString(i, end); + } else if (ch === "'") { + next = skipSingleQuoted(this.source, this.budget, i, end); + construct = this.frame('raw_string', i, next); + } else if (ch === '`') { + [construct, next] = this.parseBacktickSubstitution(i, end); + } else { + const dollar = this.parseDollar(i, end); + if (dollar === null) return null; + [construct, next] = dollar; + } + if (next >= end) return [lead, construct]; + const trail = this.frame('extglob_pattern', next, end); + return [lead, construct, trail]; + } + + /** case_item statements with the esac-argument guard active (see + * caseItemDepth). */ + private parseCaseItemStatements(): Frame[] { + this.caseItemDepth++; + try { + return this.parseStatementList({ stopWords: STOP_ESAC, stopOps: CASE_TERMINATION_OPS }); + } finally { + this.caseItemDepth--; + } + } + + /** function_definition: (`function`)? name (`()`)? body redirect? + * The body is a compound_statement, subshell, test_command or + * if_statement (the grammar's full choice). */ + private parseFunctionDefinition(): Frame { + const kids: Frame[] = []; + const first = this.lexer.next(); + if (this.tokenText(first) === 'function') { + kids.push(this.anon('function', first.start, first.end)); + const nameToken = this.lexer.peek(); + if (nameToken.type === 'word') { + this.lexer.next(); + kids.push(this.frame('word', nameToken.start, nameToken.end)); + } else { + this.hasError = true; // missing function name + } + this.consumeParenPair(kids); + } else { + kids.push(this.frame('word', first.start, first.end)); + this.consumeParenPair(kids); + } + // Blank lines/comments may sit between the header and the body. + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + break; + } + const token = this.lexer.peek(); + if (token.type === 'word' && this.tokenText(token) === '{') { + kids.push(this.parseCompoundStatement()); + } else if (token.type === 'op' && this.tokenText(token) === '(') { + kids.push(this.parseSubshell()); + } else if (token.type === 'word' && this.tokenText(token) === '[') { + kids.push(this.parseTestCommand()); + } else if (token.type === 'word' && this.tokenText(token) === 'if') { + kids.push(this.parseIfStatement()); + } else { + this.hasError = true; // missing function body + } + // Optional single redirect (file_redirect or herestring). + const redirect = this.lexer.peek(); + if (redirect.type === 'io_number') { + kids.push(this.parseRedirect()); + } else if (redirect.type === 'op') { + const op = this.tokenText(redirect); + if (isFileRedirectOp(op) || op === '<<<') { + kids.push(this.parseRedirect()); + } + } + return this.frame('function_definition', kids[0]!.start, this.endOf(kids, first.end), kids); + } + + /** Consume a `(` `)` pair (blanks allowed between) as anonymous children + * when present. */ + private consumeParenPair(kids: Frame[]): void { + const open = this.lexer.peek(); + if (open.type !== 'op' || this.tokenText(open) !== '(') return; + const close = this.lexer.peekAt(1); + if (close.type !== 'op' || this.tokenText(close) !== ')') return; + this.lexer.next(); + this.lexer.next(); + kids.push(this.anon('(', open.start, open.end)); + kids.push(this.anon(')', close.start, close.end)); + } + + /** Lookahead for the `name() body` function form: a non-reserved word + * followed by `(` `)` and a valid body start (`{`, `(`, `[`, `if`), + * possibly across newlines. Without a valid body start the tokens belong + * to a plain command (`foo () ls` is command + ERROR in the reference). */ + private isFunctionDefinitionAhead(): boolean { + const name = this.lexer.peek(); + const text = this.tokenText(name); + if (!FUNCTION_NAME_RE.test(text) || RESERVED_WORD_SET.has(text)) return false; + const open = this.lexer.peekAt(1); + if (open.type !== 'op' || this.tokenText(open) !== '(') return false; + const close = this.lexer.peekAt(2); + if (close.type !== 'op' || this.tokenText(close) !== ')') return false; + for (let n = 3; ; n++) { + const token = this.lexer.peekAt(n); + if (token.type === 'newline' || token.type === 'comment') continue; + if (token.type === 'op' && this.tokenText(token) === '(') return true; + if (token.type === 'word') { + const word = this.tokenText(token); + return word === '{' || word === '[' || word === 'if'; + } + return false; + } + } + + /** declaration_command: (`declare` | `typeset` | `export` | `readonly` | + * `local`) (variable_assignment | literal | variable_name)* */ + private parseDeclarationCommand(): Frame { + const kwToken = this.lexer.next(); + const kids: Frame[] = [this.anon(this.tokenText(kwToken), kwToken.start, kwToken.end)]; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word') break; + const text = this.tokenText(token); + if (ASSIGNMENT_RE.test(text)) { + this.lexer.next(); + kids.push(this.parseVariableAssignment(token)); + } else if (this.isSubscriptAssignmentAhead(token)) { + kids.push(this.parseSubscriptAssignment()); + } else if (IDENTIFIER_RE.test(text)) { + this.lexer.next(); + kids.push(this.frame('variable_name', token.start, token.end)); + } else { + kids.push(this.parseWordArgument()); + } + } + return this.frame('declaration_command', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + /** unset_command: (`unset` | `unsetenv`) (literal | variable_name)* */ + private parseUnsetCommand(): Frame { + const kwToken = this.lexer.next(); + const kids: Frame[] = [this.anon(this.tokenText(kwToken), kwToken.start, kwToken.end)]; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word') break; + const text = this.tokenText(token); + if (IDENTIFIER_RE.test(text)) { + this.lexer.next(); + kids.push(this.frame('variable_name', token.start, token.end)); + } else { + kids.push(this.parseWordArgument()); + } + } + return this.frame('unset_command', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + // --------------------------------------------------------------- commands + + /** + * command: leading variable_assignment / redirect prefix, command_name, + * then arguments, inline herestring redirects and a trailing subshell + * (`foo (ls)` keeps the subshell inside the command, as the reference + * does). Returns null when no command name is present and nothing was + * consumed; a nameless prefix is assembled into variable_assignment(s) / + * redirected_statement here. + */ + private parseCommand(): Frame | null { + const prefix: Frame[] = []; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word' && ASSIGNMENT_RE.test(this.tokenText(token))) { + this.lexer.next(); + prefix.push(this.parseVariableAssignment(token)); + continue; + } + if (token.type === 'word' && this.isSubscriptAssignmentAhead(token)) { + prefix.push(this.parseSubscriptAssignment()); + continue; + } + // Prefix redirects take exactly one destination so a following word + // still becomes the command_name (`> a cmd x`). Heredoc operators are + // left for the trailing-redirect path. + if (token.type === 'io_number') { + prefix.push(this.parseRedirect(1)); + continue; + } + if (token.type === 'op') { + const op = this.tokenText(token); + if (isFileRedirectOp(op) || op === '<<<') { + prefix.push(this.parseRedirect(1)); + continue; + } + } + break; + } + if (this.lexer.peek().type !== 'word') { + if (prefix.length === 0) return null; + return this.assembleNamelessPrefix(prefix); + } + const start = prefix.length > 0 ? prefix[0]!.start : this.lexer.peek().start; + // The name merges adjacent word tokens (the lexer splits { } [ ] out as + // single-character tokens): `{1..3}` or `cmd{x}` is one name. + const [nameStart, nameEnd] = this.consumeWordRun(); + const name = this.frame('command_name', nameStart, nameEnd, [this.parseLiteral(nameStart, nameEnd)]); + const command = this.frame('command', start, nameEnd, [...prefix, name]); + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word') { + // Inside a case_item, `esac` closes the item rather than becoming + // an argument (see caseItemDepth). + if (this.caseItemDepth > 0 && this.tokenText(token) === 'esac') break; + for (const argument of this.parseCommandArgument()) { + this.addKid(command, argument); + command.end = argument.end; + } + continue; + } + if (token.type === 'op' && this.tokenText(token) === '<<<') { + const herestring = this.parseHerestringRedirect(null); + this.addKid(command, herestring); + command.end = herestring.end; + continue; + } + if (token.type === 'op' && this.tokenText(token) === '(') { + const subshell = this.parseSubshell(); + this.addKid(command, subshell); + command.end = subshell.end; + continue; + } + break; + } + return command; + } + + /** Consume a run of adjacent word tokens; returns [start, end). */ + private consumeWordRun(): [number, number] { + const first = this.lexer.next(); + let end = first.end; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word' || token.start !== end) break; + this.lexer.next(); + end = token.end; + } + return [first.start, end]; + } + + /** One command argument: normally one literal, but `$"…"` (translated + * string) is TWO arguments — an anonymous `$` plus the string — because + * a bare dollar cannot start a concatenation in the reference grammar + * (tree-sitter-bash 0.25.0 never produces translated_string). */ + private parseCommandArgument(): Frame[] { + const [start, end] = this.consumeWordRun(); + if (this.source[start] === '$' && this.source[start + 1] === '"' && start + 1 < end) { + return [this.anon('$', start, start + 1), this.parseLiteral(start + 1, end)]; + } + return [this.parseLiteral(start, end)]; + } + + /** A prefix of assignments/redirects with no command name: + * `FOO=bar`, `A=1 B=2`, `> out`, `> out FOO=bar`. */ + private assembleNamelessPrefix(prefix: Frame[]): Frame { + const assignments = prefix.filter((f) => f.type === 'variable_assignment'); + if (assignments.length === prefix.length) { + if (prefix.length === 1) return prefix[0]!; + return this.frame('variable_assignments', prefix[0]!.start, prefix.at(-1)!.end, prefix); + } + return this.frame('redirected_statement', prefix[0]!.start, prefix.at(-1)!.end, prefix); + } + + /** True when a subscripted assignment (`arr[0]=x`, `a[i+1]+=2`) starts at + * the peeked word token. */ + private isSubscriptAssignmentAhead(token: Token): boolean { + if (!IDENTIFIER_RE.test(this.tokenText(token))) return false; + return SUBSCRIPT_ASSIGNMENT_RE.test(this.source.slice(token.start)); + } + + /** variable_assignment with a subscript name: `arr[0]=x`, `a[i+1]+=2`. + * The tokens are the name, `[`, index pieces, `]` and a word starting + * with the operator; ranges come from a regex over the source. */ + private parseSubscriptAssignment(): Frame { + const start = this.lexer.peek().start; + const match = SUBSCRIPT_ASSIGNMENT_RE.exec(this.source.slice(start))!; + const nameEnd = start + match[1]!.length; + const indexStart = nameEnd + 1; + const indexEnd = indexStart + match[2]!.length; + const opStart = indexEnd + 1; + const opEnd = opStart + match[3]!.length; + let valueEnd = opEnd; + // Consume tokens covering [start, opEnd) — the last one may extend past + // the operator into the value (`=x"y"` is one word token). + for (;;) { + const token = this.lexer.peek(); + if (token.start >= opEnd) break; + this.lexer.next(); + valueEnd = Math.max(valueEnd, token.end); + } + // Merge following adjacent word tokens into the value. + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word' || token.start !== valueEnd) break; + this.lexer.next(); + valueEnd = token.end; + } + const subscriptKids: Frame[] = [this.frame('variable_name', start, nameEnd), this.anon('[', nameEnd, indexStart)]; + if (indexEnd > indexStart) { + subscriptKids.push(this.parseLiteral(indexStart, indexEnd)); + } else { + this.hasError = true; // empty subscript index + } + subscriptKids.push(this.anon(']', indexEnd, opStart)); + const kids: Frame[] = [ + this.frame('subscript', start, opStart, subscriptKids), + this.anon(match[3]!, opStart, opEnd), + ]; + if (valueEnd > opEnd) { + kids.push(this.parseLiteral(opEnd, valueEnd)); + } + return this.frame('variable_assignment', start, valueEnd, kids); + } + + /** variable_assignment: NAME ( = | += ) value — the word token is split + * into variable_name, the operator, and a value parsed from the rest of + * the token plus any ADJACENT word tokens (the lexer splits { } [ ] out + * as single-character tokens, so `x={1..5}` and `x=a{b}` are single + * assignments). An empty value followed immediately by `(` is an array + * literal (`arr=(a b)`). */ + private parseVariableAssignment(token: Token): Frame { + const text = this.tokenText(token); + const match = ASSIGNMENT_SPLIT_RE.exec(text)!; + const nameEnd = token.start + match[1]!.length; + const opEnd = nameEnd + match[2]!.length; + const kids: Frame[] = [ + this.frame('variable_name', token.start, nameEnd), + this.anon(match[2]!, nameEnd, opEnd), + ]; + let valueEnd = token.end; + for (;;) { + const next = this.lexer.peek(); + if (next.type !== 'word' || next.start !== valueEnd) break; + this.lexer.next(); + valueEnd = next.end; + } + if (valueEnd > opEnd) { + kids.push(this.parseLiteral(opEnd, valueEnd)); + return this.frame('variable_assignment', token.start, valueEnd, kids); + } + const next = this.lexer.peek(); + if (next.type === 'op' && this.tokenText(next) === '(' && next.start === token.end) { + const array = this.parseArray(); + kids.push(array); + return this.frame('variable_assignment', token.start, array.end, kids); + } + return this.frame('variable_assignment', token.start, token.end, kids); + } + + /** array: `(` literal* `)` — the elements of an array assignment. */ + private parseArray(): Frame { + const open = this.lexer.next(); + const kids: Frame[] = [this.anon('(', open.start, open.end)]; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word') { + kids.push(this.parseWordArgument()); + continue; + } + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + if (token.type === 'op' && this.tokenText(token) === ')') { + this.lexer.next(); + kids.push(this.anon(')', token.start, token.end)); + return this.frame('array', open.start, token.end, kids); + } + this.hasError = true; // unterminated array + return this.frame('array', open.start, this.endOf(kids, open.end), kids); + } + } + + // --------------------------------------------------------------- redirect + + /** Parse one redirect with an optional io_number descriptor prefix already + * peeked. Dispatches to file_redirect / herestring / heredoc handling. + * `maxDestinations` limits greedy destination consumption (command-prefix + * redirects take exactly one). */ + private parseRedirect(maxDestinations = Number.POSITIVE_INFINITY): Frame { + let descriptor: Frame | null = null; + let token = this.lexer.peek(); + if (token.type === 'io_number') { + this.lexer.next(); + descriptor = this.frame('file_descriptor', token.start, token.end); + token = this.lexer.peek(); + } + if (token.type !== 'op') { + // io_number not followed by an operator — recover. + this.hasError = true; + const stray = descriptor ?? this.frame('ERROR', token.start, token.end); + return this.frame('ERROR', stray.start, stray.end, descriptor === null ? [] : [stray]); + } + const op = this.tokenText(token); + if (op === '<<' || op === '<<-') { + return this.noHeredoc ? this.parseBrokenHeredoc(descriptor) : this.parseHeredocRedirect(descriptor); + } + if (op === '<<<') return this.parseHerestringRedirect(descriptor); + if (isFileRedirectOp(op)) return this.parseFileRedirect(descriptor, maxDestinations); + this.hasError = true; + const stray = descriptor ?? this.anon(op, token.start, token.end); + return this.frame('ERROR', stray.start, stray.end, [stray]); + } + + /** file_redirect: [fd] op destination* — `>&-`/`<&-` take no destination. + * In command-prefix position only one destination is taken (`> a cmd x` + * makes `cmd` the command_name); after the command name the redirect is + * greedy, matching tree-sitter-bash (`cmd > out arg` puts both words in + * the redirect). */ + private parseFileRedirect(descriptor: Frame | null, maxDestinations = Number.POSITIVE_INFINITY): Frame { + const opToken = this.lexer.next(); + const op = this.tokenText(opToken); + const kids: Frame[] = []; + if (descriptor !== null) kids.push(descriptor); + kids.push(this.anon(op, opToken.start, opToken.end)); + let end = opToken.end; + if (op === '>&-' || op === '<&-') { + // Close-fd operators take an OPTIONAL single destination in the + // reference (`exec {a} <&- {b}>&-` puts `{b}` in the `<&-` redirect). + if (this.lexer.peek().type === 'word') { + const destination = this.parseWordArgument(); + kids.push(destination); + end = destination.end; + } + } else { + let destinations = 0; + while (destinations < maxDestinations && this.lexer.peek().type === 'word') { + const destination = this.parseWordArgument(); + kids.push(destination); + end = destination.end; + destinations++; + } + if (destinations === 0) this.hasError = true; // missing destination + } + return this.frame('file_redirect', descriptor?.start ?? opToken.start, end, kids); + } + + /** herestring_redirect: [fd] `<<<` literal */ + private parseHerestringRedirect(descriptor: Frame | null): Frame { + const opToken = this.lexer.next(); + const kids: Frame[] = []; + if (descriptor !== null) kids.push(descriptor); + kids.push(this.anon('<<<', opToken.start, opToken.end)); + let end = opToken.end; + if (this.lexer.peek().type === 'word') { + const value = this.parseWordArgument(); + kids.push(value); + end = value.end; + } else { + this.hasError = true; + } + return this.frame('herestring_redirect', descriptor?.start ?? opToken.start, end, kids); + } + + /** + * heredoc_redirect: [fd] (`<<` | `<<-`) heredoc_start, then the rest of + * the line (arguments, redirects, a pipeline or &&/|| tail, and `;`/`&` + * separated follow-up statements) absorbed into the redirect — matching + * tree-sitter-bash's grammar, which swallows all of these into + * heredoc_redirect. + * + * Only ONE heredoc may be registered per line region: a second `<<` + * (or a `<<` inside a swallowed follow-up statement) cannot be + * represented as a tree — its body would have to sit inside a node whose + * range interleaves with the first heredoc's siblings. tree-sitter-bash + * 0.25.0 errors on `cat <= MAX_PARSE_DEPTH) { + this.hasError = true; + return this.frame('ERROR', start, end); + } + this.literalDepth++; + try { + return this.parseLiteralPieces(start, end); + } finally { + this.literalDepth--; + } + } + + private parseLiteralPieces(start: number, end: number): Frame { + const pieces: Frame[] = []; + let i = start; + while (i < end) { + this.budget.progress(); + const ch = this.source[i]!; + if (ch === '"') { + const [piece, next] = this.parseString(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === "'") { + const close = skipSingleQuoted(this.source, this.budget, i, end); + if (close >= end && this.source[close - 1] !== "'") this.hasError = true; // unterminated raw string + pieces.push(this.frame('raw_string', i, close)); + i = close; + continue; + } + if (ch === '`') { + const [piece, next] = this.parseBacktickSubstitution(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === '$') { + // $"…" at the START of a literal is a translated_string (the + // reference produces it everywhere except command-argument + // position, where parseCommandArgument splits the bare `$` and the + // string into two arguments). Mid-literal it stays a bare `$` plus + // a string piece inside the concatenation. + if (this.source[i + 1] === '"' && i === start) { + const [translated, next] = this.parseString(i + 1, end); + pieces.push(this.frame('translated_string', i, next, [this.anon('$', i, i + 1), translated])); + i = next; + continue; + } + const dollar = this.parseDollar(i, end); + if (dollar !== null) { + pieces.push(dollar[0]); + i = dollar[1]; + } else { + // Bare `$` (not followed by anything expandable), e.g. the `$` of + // a mid-literal $"…" (see above). + pieces.push(this.anon('$', i, i + 1)); + i++; + } + continue; + } + if ((ch === '<' || ch === '>') && this.source[i + 1] === '(' && i + 1 < end) { + const [piece, next] = this.parseProcessSubstitution(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === '(' && this.source[i + 1] === '(' && i + 1 < end) { + const [piece, next] = this.parseParenArithmetic(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === '{') { + const brace = BRACE_EXPRESSION_RE.exec(this.source.slice(i, end)); + if (brace !== null) { + const [full, low, high] = brace; + const lowStart = i + 1; + const highStart = lowStart + low!.length + 2; + const close = highStart + high!.length; + pieces.push( + this.frame('brace_expression', i, i + full.length, [ + this.anon('{', i, lowStart), + this.frame('number', lowStart, lowStart + low!.length), + this.anon('..', lowStart + low!.length, highStart), + this.frame('number', highStart, close), + this.anon('}', close, close + 1), + ]), + ); + i += full.length; + continue; + } + pieces.push(this.frame('word', i, i + 1)); + i++; + continue; + } + if (ch === '}' || ch === '[' || ch === ']') { + pieces.push(this.frame('word', i, i + 1)); + i++; + continue; + } + // Bare run of ordinary word characters (escapes included as-is). + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const next = this.source[j]!; + if (next === '"' || next === "'" || next === '`' || next === '$') break; + if (next === '{' || next === '}' || next === '[' || next === ']') break; + if ((next === '<' || next === '>') && this.source[j + 1] === '(' && j + 1 < end) break; + if (next === '(' && this.source[j + 1] === '(' && j + 1 < end) break; + if (next === '\\' && j + 1 < end) { + j += 2; + continue; + } + j++; + } + if (j === i) j++; // defensive: never stall + // A bare run that is exactly a number is a number node even as one + // piece of a concatenation (`a[2]b` → word + number + word). + const numeric = NUMBER_RE.test(this.text(i, j)); + pieces.push(this.frame(numeric ? 'number' : 'word', i, j)); + i = j; + } + if (pieces.length === 1) { + const only = pieces[0]!; + if (only.type === 'word' && NUMBER_RE.test(this.text(only.start, only.end))) { + return this.frame('number', only.start, only.end); + } + return only; + } + return this.frame('concatenation', start, end, pieces); + } + + /** string: " … " with string_content chunks and expansions inside. */ + private parseString(start: number, rangeEnd: number): [Frame, number] { + const string = this.frame('string', start, start + 1, [this.anon('"', start, start + 1)]); + let chunkStart = start + 1; + let i = start + 1; + const flushChunk = (upto: number): void => { + if (upto > chunkStart) { + this.addKid(string, this.frame('string_content', chunkStart, upto)); + } + }; + /** A whitespace-only chunk is absorbed into the next construct's opener + * (` " ${x}"` → (${) token " ${"), or into the closing quote after an + * expansion (`"$x "` → (") token " \"") — the reference scanner never + * emits a whitespace-only string_content at those positions. Leaf + * constructs without an opener token (ansi_c_string) cannot absorb. */ + const whitespaceOnlyChunk = (upto: number): boolean => { + if (upto <= chunkStart) return false; + return /^[ \t\r]+$/.test(this.source.slice(chunkStart, upto)); + }; + const absorbInto = (piece: Frame): boolean => { + const first = piece.children[0]; + if (first === undefined) return false; + piece.start = chunkStart; + first.start = chunkStart; + return true; + }; + let sinceTick = 0; + while (i < rangeEnd) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[i]!; + if (ch === '"') { + const last = string.children.at(-1); + if (whitespaceOnlyChunk(i) && last !== undefined && last.isNamed && last.type !== 'string_content') { + // Absorbed into the closing quote (see whitespaceOnlyChunk). + this.addKid(string, this.anon('"', chunkStart, i + 1)); + } else { + flushChunk(i); + this.addKid(string, this.anon('"', i, i + 1)); + } + string.end = i + 1; + return [string, i + 1]; + } + if (ch === '\\') { + i += 2; // escaped characters stay inside the current content chunk + continue; + } + if (ch === '$') { + const dollar = this.parseDollar(i, rangeEnd); + if (dollar !== null) { + if (!(whitespaceOnlyChunk(i) && absorbInto(dollar[0]))) { + flushChunk(i); + } + this.addKid(string, dollar[0]); + i = dollar[1]; + chunkStart = i; + continue; + } + // A `$` that starts no construct is an anonymous ($) token, splitting + // the content (`"s$"` → string_content "s", ($) "$"). + flushChunk(i); + this.addKid(string, this.anon('$', i, i + 1)); + i++; + chunkStart = i; + continue; + } + if (ch === '`') { + flushChunk(i); + const [piece, next] = this.parseBacktickSubstitution(i, rangeEnd); + this.addKid(string, piece); + i = next; + chunkStart = i; + continue; + } + i++; + } + // Unterminated string: keep the partial node, flag the error. + this.hasError = true; + flushChunk(rangeEnd); + string.end = rangeEnd; + return [string, rangeEnd]; + } + + /** Dispatch a $-construct at `i`: simple_expansion, expansion, + * command_substitution, arithmetic_expansion, or ansi_c_string. Returns + * null for a bare `$` (including the `$` of a mid-literal $"…" — see + * parseLiteralPieces for the translated_string rule). */ + private parseDollar(i: number, rangeEnd: number): [Frame, number] | null { + const next = this.source[i + 1]; + if (next === '(' && i + 1 < rangeEnd) { + if (this.source[i + 2] === '(') return this.parseArithmeticExpansion(i, rangeEnd); + return this.parseCommandSubstitution(i, rangeEnd); + } + if (next === '{' && i + 1 < rangeEnd) return this.parseExpansion(i, rangeEnd); + if (next === '[' && i + 1 < rangeEnd) return this.parseBracketArithmetic(i, rangeEnd); + if (next === "'" && i + 1 < rangeEnd) return this.parseAnsiCString(i, rangeEnd); + if (next !== undefined && /[\w]/.test(next)) { + let j = i + 1; + while (j < rangeEnd && /[\w]/.test(this.source[j]!)) j++; + // `$0` is a special_variable_name; other digits are variable_name + // (`$1`, and `${10}` below) — matching the reference grammar, whose + // \w+ rule wins for everything except the standalone 0. + if (j === i + 2 && next === '0') { + return [ + this.frame('simple_expansion', i, j, [this.anon('$', i, i + 1), this.frame('special_variable_name', i + 1, j)]), + j, + ]; + } + const expansion = this.frame('simple_expansion', i, j, [ + this.anon('$', i, i + 1), + this.frame('variable_name', i + 1, j), + ]); + return [expansion, j]; + } + if (next !== undefined && i + 1 < rangeEnd && SPECIAL_VARIABLE_CHARS.includes(next)) { + const expansion = this.frame('simple_expansion', i, i + 2, [ + this.anon('$', i, i + 1), + this.frame('special_variable_name', i + 1, i + 2), + ]); + return [expansion, i + 2]; + } + return null; + } + + /** ansi_c_string: $' … ' — one leaf node, \' is an escaped quote. */ + private parseAnsiCString(i: number, rangeEnd: number): [Frame, number] { + let j = i + 2; + let sinceTick = 0; + while (j < rangeEnd) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === "'") { + return [this.frame('ansi_c_string', i, j + 1), j + 1]; + } + j++; + } + this.hasError = true; // unterminated ANSI-C string + return [this.frame('ansi_c_string', i, rangeEnd), rangeEnd]; + } + + /** + * expansion: ${ … } with optional prefix operators (! #), a variable_name + * / special_variable_name / subscript, and an optional infix operator + * whose value layout follows the reference: + * # ## % %% removal — the pattern is a regex node (single-quoted + * segments fold into it; double quotes stay strings) + * / // /# /% replacement — regex pattern, optional `/` + literal + * ^ ^^ , ,, case modification — optional regex + * @X transformation — two anonymous operator nodes + * : max length — arithmetic values around an optional `:` + * :- := :+ :? - = + ? default/assign/… — a word/concatenation value + * + * Guarded by literalDepth directly (not only via parseLiteral): the + * pattern → string → $ → expansion recursion cycle bypasses parseLiteral + * and would otherwise overflow the stack on pathological nesting. + */ + private parseExpansion(i: number, rangeEnd: number): [Frame, number] { + if (this.literalDepth >= MAX_PARSE_DEPTH) { + this.hasError = true; + const scan = this.scanBalanced(i + 1, rangeEnd, '{', '}'); + return [this.frame('ERROR', i, scan.end), scan.end]; + } + this.literalDepth++; + try { + return this.parseExpansionInner(i, rangeEnd); + } finally { + this.literalDepth--; + } + } + + /** + * Whether a ${a[…]} subscript index parses as an expression in the + * reference (binary/unary/parenthesized) rather than as a plain literal: + * true when the index contains a top-level arithmetic operator ADJACENT + * TO WHITESPACE (`${a[1 + 2]}` — a fused `i+1` stays a word) or starts + * with ++/-- (`${w[++counter]}`). Operators inside parens, brackets, + * quotes and $-constructs do not count. + */ + private isArithmeticSubscriptIndex(start: number, end: number): boolean { + if (this.source.startsWith('++', start) || this.source.startsWith('--', start)) return true; + const isBlank = (ch: string | undefined): boolean => ch === ' ' || ch === '\t' || ch === '\r'; + let i = start; + while (i < end) { + const ch = this.source[i]!; + if (ch === '\\') { + i += 2; + continue; + } + if (ch === '"' || ch === "'" || ch === '`') { + i++; + continue; + } + if (ch === '$') { + i = this.skipDollarConstruct(i, end); + continue; + } + if (ch === '(') { + i = this.scanBalanced(i, end, '(', ')').end; + continue; + } + if (ch === '[') { + i = this.scanBalanced(i, end, '[', ']').end; + continue; + } + if ('+-*/%<>=!&|^~?,'.includes(ch) && (isBlank(this.source[i - 1]) || isBlank(this.source[i + 1]))) return true; + i++; + } + return false; + } + + /** Rename variable_name leaves to word (the reference uses plain _literal + * operands — word, not variable_name — in subscript index expressions). */ + private remapVariableNamesToWords(frame: Frame): void { + if (frame.type === 'variable_name') frame.type = 'word'; + for (const child of frame.children) this.remapVariableNamesToWords(child); + } + + private parseExpansionInner(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '{', '}'); + const close = scan.end; + const terminated = scan.balanced; + if (!terminated) this.hasError = true; + const innerEnd = terminated ? close - 1 : close; + const expansion = this.frame('expansion', i, close, [this.anon('${', i, i + 2)]); + let j = i + 2; + // Prefix operators: ${#v} (length), ${!v} (indirect). + let bangPrefix = false; + while (j < innerEnd && (this.source[j] === '#' || this.source[j] === '!')) { + const after = this.source[j + 1]; + if (after === undefined || j + 1 >= innerEnd || !/[\w@*#?$!-]/.test(after)) break; + if (this.source[j] === '!') bangPrefix = true; + this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); + j++; + } + // The variable itself. + let hasName = false; + if (j < innerEnd && /[\w]/.test(this.source[j]!)) { + let nameEnd = j; + while (nameEnd < innerEnd && /[\w]/.test(this.source[nameEnd]!)) nameEnd++; + // ${0} is a special_variable_name; other digits are variable_name. + const special = nameEnd === j + 1 && this.source[j] === '0'; + this.addKid(expansion, this.frame(special ? 'special_variable_name' : 'variable_name', j, nameEnd)); + j = nameEnd; + hasName = true; + } else if (j < innerEnd && (this.source[j] === '#' || this.source[j] === '!') && j + 1 === innerEnd) { + // A lone # / ! in variable position is an anonymous token in the + // reference (`${#}`, `${!#}`, `${!##}`, `${#!}` — the length/indirect + // operator with no name), not a special_variable_name. + this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); + j++; + } else if (j < innerEnd && SPECIAL_VARIABLE_CHARS.includes(this.source[j]!)) { + this.addKid(expansion, this.frame('special_variable_name', j, j + 1)); + j++; + hasName = true; + } + // Subscript: ${a[0]} — replaces the bare variable_name child. A simple + // index is a literal; an index with a top-level arithmetic operator or a + // ++/-- prefix is an expression in the reference (`${a[1 + 2]}` → + // binary_expression, `${w[++c]}` → unary_expression). + if (j < innerEnd && this.source[j] === '[' && hasName) { + const sub = this.scanBalanced(j, rangeEnd, '[', ']'); + const subEnd = sub.end; + if (!sub.balanced) this.hasError = true; + const indexEnd = sub.balanced ? subEnd - 1 : subEnd; + const variable = expansion.children.pop()!; + const subscript = this.frame('subscript', variable.start, subEnd, [ + variable, + this.anon('[', j, j + 1), + ]); + if (indexEnd > j + 1) { + const indexText = this.text(j + 1, indexEnd); + if (indexText === '*' || indexText === '@') { + // ${a[*]} / ${a[@]} — all-indices subscripts are plain words. + this.addKid(subscript, this.parseLiteral(j + 1, indexEnd)); + } else if (this.isArithmeticSubscriptIndex(j + 1, indexEnd)) { + const st = this.newExprState(j + 1, indexEnd, 'arith'); + const index = this.parseExpression(st, 0); + if (index !== null) { + this.remapVariableNamesToWords(index); + this.addKid(subscript, index); + } + const leftover = this.exprLeftover(st); + if (leftover !== null) this.addKid(subscript, leftover); + } else { + this.addKid(subscript, this.parseLiteral(j + 1, indexEnd)); + } + } else { + this.hasError = true; + } + if (sub.balanced) this.addKid(subscript, this.anon(']', subEnd - 1, subEnd)); + this.addKid(expansion, subscript); + j = subEnd; + } + // ${!prefix*} / ${!name@}: a trailing * or @ after an indirect name. + if (bangPrefix && j < innerEnd && (this.source[j] === '*' || this.source[j] === '@')) { + this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); + j++; + } + // Infix operator plus an optional value. + if (j < innerEnd) { + j = this.parseExpansionInfix(expansion, j, innerEnd); + } + if (terminated) this.addKid(expansion, this.anon('}', close - 1, close)); + return [expansion, close]; + } + + /** The infix part of an expansion (see parseExpansion). Returns the new + * scan position. */ + private parseExpansionInfix(expansion: Frame, j: number, innerEnd: number): number { + const one = this.source[j]!; + const two = this.source.slice(j, j + 2); + if (two === '##' || two === '%%' || one === '#' || one === '%') { + const operator = two === '##' || two === '%%' ? two : one; + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + return this.parseExpansionPattern(expansion, j + operator.length, innerEnd); + } + if (two === '//' || two === '/#' || two === '/%' || one === '/') { + const operator = two === '//' || two === '/#' || two === '/%' ? two : one; + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + let pos = this.parseExpansionPattern(expansion, j + operator.length, innerEnd, '/'); + if (pos < innerEnd && this.source[pos] === '/') { + this.addKid(expansion, this.anon('/', pos, pos + 1)); + pos++; + if (pos < innerEnd) { + for (const piece of this.parseExpansionValue(pos, innerEnd)) this.addKid(expansion, piece); + pos = innerEnd; + } + } + return pos; + } + if (two === '^^' || two === ',,' || one === '^' || one === ',') { + const operator = two === '^^' || two === ',,' ? two : one; + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + return this.parseExpansionPattern(expansion, j + operator.length, innerEnd); + } + if (one === '@' && j + 1 < innerEnd) { + this.addKid(expansion, this.anon('@', j, j + 1)); + this.addKid(expansion, this.anon(this.source[j + 1]!, j + 1, j + 2)); + return j + 2; + } + if (one === ':' && two !== ':-' && two !== ':=' && two !== ':+' && two !== ':?') { + // Max length: ${v:offset:length} — arithmetic values. + this.addKid(expansion, this.anon(':', j, j + 1)); + let pos = this.parseMaxLengthValue(expansion, j + 1, innerEnd); + if (pos < innerEnd && this.source[pos] === ':') { + this.addKid(expansion, this.anon(':', pos, pos + 1)); + pos = this.parseMaxLengthValue(expansion, pos + 1, innerEnd); + } + if (pos < innerEnd) { + this.hasError = true; // extra content after ${v:o:l} + this.addKid(expansion, this.parseLiteral(pos, innerEnd)); + pos = innerEnd; + } + return pos; + } + for (const operator of [':-', ':=', ':+', ':?', '-', '=', '+', '?']) { + if (this.source.startsWith(operator, j) && j + operator.length <= innerEnd) { + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + const pos = j + operator.length; + if (pos < innerEnd) { + for (const piece of this.parseExpansionValue(pos, innerEnd)) this.addKid(expansion, piece); + } + return innerEnd; + } + } + // Unknown infix content: keep it as a literal (and flag the error). + this.hasError = true; + this.addKid(expansion, this.parseLiteral(j, innerEnd)); + return innerEnd; + } + + /** A removal/replacement pattern inside ${…}: regex chunks interleaved + * with string pieces. Rules from the reference scanner: + * - leading whitespace after the operator drops out of the tree when + * more pattern follows (${G%% *} → regex "*"); an all-whitespace + * pattern stays a regex (${B[0]# } → regex " "); + * - in the REPLACEMENT pattern (before the separator `/`), single-quoted + * segments fold INTO the surrounding regex chunk (${M/'N'X/O} → + * regex "'N'X"); in removal patterns (# ## % %% ^ ^ , ,,) they stay + * separate raw_string nodes, and double quotes always stay strings; + * - the replacement separator `/` ends the pattern even when escaped + * (a pattern ending in backslash-slash → regex keeps the backslash, + * then the `/` operator). */ + private parseExpansionPattern(expansion: Frame, j: number, innerEnd: number, stop?: string): number { + let pos = j; + while (pos < innerEnd && (this.source[pos] === ' ' || this.source[pos] === '\t' || this.source[pos] === '\r')) { + pos++; + } + if (pos > j && (pos >= innerEnd || (stop !== undefined && this.source[pos] === stop))) { + // An all-whitespace pattern stays a regex node. + this.addKid(expansion, this.frame('regex', j, pos)); + return pos; + } + const mergeQuotes = stop === '/'; + while (pos < innerEnd) { + this.budget.progress(); + const ch = this.source[pos]!; + if (ch === '"') { + const [piece, next] = this.parseString(pos, innerEnd); + this.addKid(expansion, piece); + pos = next; + continue; + } + if (ch === "'" && !mergeQuotes) { + const close = skipSingleQuoted(this.source, this.budget, pos, innerEnd); + this.addKid(expansion, this.frame('raw_string', pos, close)); + pos = close; + continue; + } + // One regex chunk: bare text, plus single-quoted segments when they + // fold into the regex (replacement patterns). + let k = pos; + let sinceTick = 0; + while (k < innerEnd && this.source[k] !== '"' && this.source[k] !== stop) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ck = this.source[k]!; + if (ck === "'") { + if (!mergeQuotes) break; + k = skipSingleQuoted(this.source, this.budget, k, innerEnd); + continue; + } + if (ck === '\\' && k + 1 < innerEnd) { + if (stop !== undefined && this.source[k + 1] === stop) { + k++; // the escape backslash stays in the chunk; the stop char ends it + break; + } + k += 2; + continue; + } + k++; + } + if (k === pos) break; // defensive: only the stop character remains + this.addKid(expansion, this.frame('regex', pos, k)); + pos = k; + } + return pos; + } + + /** A default/assign value inside ${…} (`${v:-word}`), as a list of frames + * (usually one — the reference sometimes attaches the value as several + * direct children of the expansion). Layout rules from the reference: + * - a bare value is a word (never a number — `${v:-1}` is a word). An + * interior space splits it into word + word (the second keeps the + * leading space: `${v:-d e f}` → word "d" + word " e f"); trailing-only + * whitespace stays inside the single word (`${v:-) }` → word ") "), and + * a leading `? ` (Gentoo atom syntax) keeps the whole value as one word; + * - bare word pieces directly before a quote/construct are right-trimmed + * (`${v:-command '$1' x}` → word "command", raw_string, word " x") and + * a whitespace-only piece before a construct drops out entirely + * (`${2+ ${2}}` → just the expansion); + * - a command_substitution followed by a bare remainder attaches as two + * children without a concatenation wrapper. */ + private parseExpansionValue(start: number, end: number): Frame[] { + const raw = this.text(start, end); + if (!/["'$`\\]/.test(raw)) { + // Bare value. Rules from the reference's _expansion_word scanner: + // - an interior space splits it into word + word (the second keeps + // the leading space: `${v:-d e f}` → word "d" + word " e f"); + // - leading or trailing whitespace stays inside the single word + // (`${v:- -quiet}` → word " -quiet", `${v:-) }` → word ") "), and a + // leading `? ` (Gentoo atom syntax) keeps the whole value as one + // word. Whitespace only drops out directly before a quote or a + // construct (handled by the piece path below). + if (this.source.startsWith('? ', start)) return [this.frame('word', start, end)]; + for (let k = start + 1; k < end; k++) { + const ch = this.source[k]!; + if (ch !== ' ' && ch !== '\t' && ch !== '\r') continue; + // Split at the first interior space (non-space text follows it). + if (/[^ \t\r]/.test(this.source.slice(k + 1, end))) { + return [ + this.frame('concatenation', start, end, [ + this.frame('word', start, k), + this.frame('word', k, end), + ]), + ]; + } + break; + } + return [this.frame('word', start, end)]; + } + const value = this.parseLiteral(start, end); + const pieces = value.type === 'concatenation' ? [...value.children] : [value]; + // Right-trim bare word pieces that directly precede another piece (the + // whitespace drops out of the tree); drop pieces that become empty. + // Pieces starting with `(` keep their spaces (the reference's + // expansion_word scanner treats them via its paren branch). + const trimmed: Frame[] = []; + for (let p = 0; p < pieces.length; p++) { + const piece = pieces[p]!; + if (piece.type === 'word' && p + 1 < pieces.length && this.source[piece.start] !== '(') { + let newEnd = piece.end; + while (newEnd > piece.start && (this.source[newEnd - 1] === ' ' || this.source[newEnd - 1] === '\t' || this.source[newEnd - 1] === '\r')) { + newEnd--; + } + if (newEnd === piece.start) continue; + trimmed.push(newEnd === piece.end ? piece : this.frame('word', piece.start, newEnd)); + } else { + trimmed.push(piece); + } + } + // command_substitution + bare remainder: two direct children. + if (trimmed.length === 2 && trimmed[0]!.type === 'command_substitution') return trimmed; + if (trimmed.length === 1) return trimmed; + if (trimmed.length === 0) return []; + return [this.frame('concatenation', start, end, trimmed)]; + } + + /** One arithmetic value of ${v:offset:length} (up to `:` or innerEnd). */ + private parseMaxLengthValue(expansion: Frame, j: number, innerEnd: number): number { + let end = j; + let sinceTick = 0; + while (end < innerEnd && this.source[end] !== ':') { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[end]!; + if (ch === '\\') { + end += 2; + continue; + } + if (ch === '$') { + end = this.skipDollarConstruct(end, innerEnd); + continue; + } + if (ch === '(') { + end = this.scanBalanced(end, innerEnd, '(', ')').end; + continue; + } + end++; + } + if (end > j) { + // In max-length context a negative offset is a number literal, not a + // unary expression (`${x: -5}` → number "-5" in the reference). + const negative = /^\s*(-\d+)\s*$/.exec(this.text(j, end)); + if (negative !== null) { + const start = j + this.text(j, end).indexOf('-'); + this.addKid(expansion, this.frame('number', start, start + negative[1]!.length)); + return end; + } + const st = this.newExprState(j, end, 'arith'); + const value = this.parseExpression(st, 0); + if (value !== null) { + this.addKid(expansion, value); + } + const leftover = this.exprLeftover(st); + if (leftover !== null) this.addKid(expansion, leftover); + } + return end; + } + + /** Skip a $-construct without building nodes (max-length scanning). */ + private skipDollarConstruct(i: number, end: number): number { + const next = this.source[i + 1]; + if (next === '(') return this.scanBalanced(i + 1, end, '(', ')').end; + if (next === '{') return this.scanBalanced(i + 1, end, '{', '}').end; + if (next !== undefined && /[\w]/.test(next)) { + let j = i + 1; + while (j < end && /[\w]/.test(this.source[j]!)) j++; + return j; + } + return i + 1 < end ? i + 2 : i + 1; + } + + /** command_substitution: $( _statements ) — or the special $( redirect ) + * form (`$(< file)`), which holds a bare file_redirect without the + * redirected_statement wrapper in the reference. The close-paren scan is + * case-aware (see scanBalancedStatements): a case_item pattern `)` must + * not close the substitution early. */ + private parseCommandSubstitution(i: number, rangeEnd: number): [Frame, number] { + const scan = scanBalancedStatements(this.source, this.budget, i + 1, rangeEnd); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + const innerEnd = scan.balanced ? close - 1 : close; + const substitution = this.frame('command_substitution', i, close, [this.anon('$(', i, i + 2)]); + let children = this.parseScopedStatements(i + 2, innerEnd); + if ( + children.length === 1 && + children[0]!.type === 'redirected_statement' && + children[0]!.children.length > 0 && + children[0]!.children.every((child) => child.type === 'file_redirect' || child.type === 'herestring_redirect') + ) { + children = children[0]!.children; + } + for (const child of children) { + this.addKid(substitution, child); + } + if (scan.balanced) this.addKid(substitution, this.anon(')', close - 1, close)); + return [substitution, close]; + } + + /** command_substitution: ` _statements ` (legacy backtick form). */ + private parseBacktickSubstitution(i: number, rangeEnd: number): [Frame, number] { + const close = skipBacktick(this.source, this.budget, i, rangeEnd); + const terminated = close > i + 1 && this.source[close - 1] === '`'; + if (!terminated) this.hasError = true; + const innerEnd = terminated ? close - 1 : close; + const substitution = this.frame('command_substitution', i, close, [this.anon('`', i, i + 1)]); + if (innerEnd > i + 1) { + for (const child of this.parseScopedStatements(i + 1, innerEnd)) { + this.addKid(substitution, child); + } + } + if (terminated) this.addKid(substitution, this.anon('`', close - 1, close)); + return [substitution, close]; + } + + /** process_substitution: ( <( | >( ) _statements ) — case-aware close + * scan, like command_substitution. */ + private parseProcessSubstitution(i: number, rangeEnd: number): [Frame, number] { + const scan = scanBalancedStatements(this.source, this.budget, i + 1, rangeEnd); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + const innerEnd = scan.balanced ? close - 1 : close; + const opener = this.source[i]!; + const substitution = this.frame('process_substitution', i, close, [this.anon(`${opener}(`, i, i + 2)]); + for (const child of this.parseScopedStatements(i + 2, innerEnd)) { + this.addKid(substitution, child); + } + if (scan.balanced) this.addKid(substitution, this.anon(')', close - 1, close)); + return [substitution, close]; + } + + // ------------------------------------------------------ expression engine + + private newExprState(start: number, end: number, mode: ExprMode): ExprState { + return { pos: start, end, mode, lookahead: null, parenDepth: 0, expectOperator: false }; + } + + private exprPeek(st: ExprState): ExprToken { + st.lookahead ??= this.scanExprToken(st); + return st.lookahead; + } + + /** When unconsumed input remains in the expression state, wrap it in an + * ERROR node (covering the rest of the region, so no source text + * silently drops out of the tree) and flag the error. */ + private exprLeftover(st: ExprState): Frame | null { + const token = this.exprPeek(st); + if (token.kind === 'end') return null; + this.hasError = true; + return this.frame('ERROR', token.start, st.end); + } + + private exprNext(st: ExprState): ExprToken { + const token = this.exprPeek(st); + st.lookahead = null; + return token; + } + + /** arithmetic_expansion: $(( expr (, expr)* )) */ + private parseArithmeticExpansion(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); + const close = scan.end; + // The content sits between `$((` and the final `))`. + const closed = scan.balanced && close - 2 >= i + 3 && this.source[close - 2] === ')'; + if (!closed) this.hasError = true; + const innerStart = Math.min(i + 3, close); + const innerEnd = closed ? close - 2 : close; + const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('$((', i, innerStart)]); + this.addArithmeticChildren(expansion, innerStart, innerEnd); + if (closed) { + this.addKid(expansion, this.anon('))', innerEnd, close)); + } + return [expansion, close]; + } + + /** arithmetic_expansion without the `$`: (( expr )) at word position. */ + private parseParenArithmetic(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i, rangeEnd, '(', ')'); + const close = scan.end; + const closed = scan.balanced && close - 2 >= i + 2 && this.source[close - 2] === ')'; + if (!closed) this.hasError = true; + const innerStart = Math.min(i + 2, close); + const innerEnd = closed ? close - 2 : close; + const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('((', i, innerStart)]); + this.addArithmeticChildren(expansion, innerStart, innerEnd); + if (closed) { + this.addKid(expansion, this.anon('))', innerEnd, close)); + } + return [expansion, close]; + } + + /** arithmetic_expansion: $[ expr ] (legacy form). */ + private parseBracketArithmetic(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '[', ']'); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + const innerEnd = scan.balanced ? close - 1 : close; + const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('$[', i, i + 2)]); + this.addArithmeticChildren(expansion, i + 2, innerEnd); + if (scan.balanced) { + this.addKid(expansion, this.anon(']', close - 1, close)); + } + return [expansion, close]; + } + + /** The comma-separated expression list inside an arithmetic_expansion. */ + private addArithmeticChildren(expansion: Frame, start: number, end: number): void { + const st = this.newExprState(start, end, 'arith'); + for (;;) { + const expression = this.parseExpression(st, 0); + if (expression !== null) this.addKid(expansion, expression); + const token = this.exprPeek(st); + if (token.kind === 'op' && token.text === ',') { + this.exprNext(st); + this.addKid(expansion, this.anon(',', token.start, token.end)); + continue; + } + break; + } + const leftover = this.exprLeftover(st); + if (leftover !== null) this.addKid(expansion, leftover); + } + + /** + * Pratt parser for arithmetic / c-style-for / test expressions. Mirrors + * grammar.js: left-associative binaries parse their right side at + * level+1, `**` is right-associative only in test mode (left in + * arithmetic), prefix -/+/~/! take their operand at level 11 (so + * `-x + ~y` is -(x + ~y)) and prefix ++/-- at level 17, postfix ++/-- sit + * at level 18, ternary at level 2. + */ + private parseExpression(st: ExprState, minPrecedence: number): Frame | null { + let left: Frame | null; + const head = this.exprPeek(st); + // `;` and `,` are separators (c-style for parts, comma lists), handled + // by the caller — never operands. + if (head.kind === 'op' && (head.text === ';' || head.text === ',')) return null; + if (head.kind === 'op' && (head.text === '++' || head.text === '--')) { + this.exprNext(st); + const kids: Frame[] = [this.anon(head.text, head.start, head.end)]; + const operand = this.parseExpression(st, PREC_PREFIX); + if (operand === null) this.hasError = true; + else kids.push(operand); + left = this.frame('unary_expression', head.start, this.endOf(kids, head.end), kids); + } else if (head.kind === 'op' && (head.text === '!' || head.text === '~' || head.text === '+' || head.text === '-')) { + this.exprNext(st); + const kids: Frame[] = [this.anon(head.text, head.start, head.end)]; + const operand = this.parseExpression(st, PREC_UNARY); + if (operand === null) this.hasError = true; + else kids.push(operand); + left = this.frame('unary_expression', head.start, this.endOf(kids, head.end), kids); + } else if (head.kind === 'testop') { + this.exprNext(st); + // A test_operator with no usable operand ahead is just a word + // (`[[ $a == -foo ]]` → word "-foo", `[[ -foo == x ]]` → word, + // `[[ -f ]]` → word — the reference's fallback when the operator + // reading fails). + const after = this.exprPeek(st); + const demote = + after.kind === 'end' || + after.kind === 'rparen' || + (after.kind === 'op' && + (after.text === '=' || after.text === '==' || after.text === '!=' || after.text === '=~' || after.text === '&&' || after.text === '||')); + if (demote) { + left = this.frame('word', head.start, head.end); + } else { + const kids: Frame[] = [this.frame('test_operator', head.start, head.end)]; + const operand = this.parseExpression(st, PREC_TEST); + if (operand === null) this.hasError = true; + else kids.push(operand); + left = this.frame('unary_expression', head.start, this.endOf(kids, head.end), kids); + } + } else { + left = this.parseExprPrimary(st); + } + if (left === null) return null; + // A complete operand: what follows is operator position (test mode). + st.expectOperator = true; + for (;;) { + const token = this.exprPeek(st); + if (token.kind === 'op' && (token.text === '++' || token.text === '--') && PREC_POSTFIX >= minPrecedence) { + this.exprNext(st); + left = this.frame('postfix_expression', left.start, token.end, [ + left, + this.anon(token.text, token.start, token.end), + ]); + continue; + } + if (token.kind === 'op' && token.text === '?' && PREC_TERNARY >= minPrecedence) { + this.exprNext(st); + st.expectOperator = false; + const kids: Frame[] = [left, this.anon('?', token.start, token.end)]; + const consequence = this.parseExpression(st, 0); + if (consequence === null) this.hasError = true; + else kids.push(consequence); + const colon = this.exprPeek(st); + if (colon.kind === 'op' && colon.text === ':') { + this.exprNext(st); + st.expectOperator = false; + kids.push(this.anon(':', colon.start, colon.end)); + } else { + this.hasError = true; // missing : in ternary + } + const alternative = this.parseExpression(st, PREC_TERNARY + 1); + if (alternative === null) this.hasError = true; + else kids.push(alternative); + left = this.frame('ternary_expression', left.start, this.endOf(kids, token.end), kids); + st.expectOperator = true; + continue; + } + const isTestOp = token.kind === 'testop'; + const precedence = isTestOp ? PREC_TEST : token.kind === 'op' ? EXPRESSION_PRECEDENCE[token.text] : undefined; + if (precedence === undefined || precedence < minPrecedence) break; + this.exprNext(st); + st.expectOperator = false; + // `=~` in a test command takes a raw regex (or string/expansion) as + // its right side instead of a full expression. + if (st.mode === 'test' && token.kind === 'op' && token.text === '=~') { + const right = this.parseTestRegex(st); + const kids: Frame[] = [left, this.anon('=~', token.start, token.end)]; + if (right === null) this.hasError = true; + else kids.push(...right); + left = this.frame('binary_expression', left.start, this.endOf(kids, token.end), kids); + st.expectOperator = true; + continue; + } + const rightPrecedence = token.text === '**' && st.mode === 'test' ? precedence : precedence + 1; + let right: Frame | null; + // A pattern-shaped right side after ==/!= (extglob group, or glob + // text mixed with a quote/expansion) — see tryParseTestPattern. + if (st.mode === 'test' && (token.text === '==' || token.text === '!=')) { + const pattern = this.tryParseTestPattern(st); + if (pattern !== null) { + const kids: Frame[] = [left, this.anon(token.text, token.start, token.end), ...pattern.frames]; + left = this.frame('binary_expression', left.start, pattern.end, kids); + st.expectOperator = true; + continue; + } + } + right = this.parseExpression(st, rightPrecedence); + if (right === null) { + this.hasError = true; // missing right operand + } else if (st.mode === 'test' && token.kind === 'op') { + right = this.convertTestRightSide(token.text, right, st); + } + const operator = + token.kind === 'testop' + ? this.frame('test_operator', token.start, token.end) + : this.anon(token.text, token.start, token.end); + const kids: Frame[] = right === null ? [left, operator] : [left, operator, right]; + left = this.frame('binary_expression', left.start, this.endOf(kids, token.end), kids); + st.expectOperator = true; + } + return left; + } + + private parseExprPrimary(st: ExprState): Frame | null { + const token = this.exprPeek(st); + switch (token.kind) { + case 'end': + case 'rparen': + return null; + case 'number': + this.exprNext(st); + return this.frame('number', token.start, token.end); + case 'ident': { + this.exprNext(st); + if (st.mode === 'c') { + // `name = value` in a c-style for header is a variable_assignment + // (all other assignment operators are binary_expressions). + const next = this.exprPeek(st); + if (next.kind === 'op' && next.text === '=') { + this.exprNext(st); + const kids: Frame[] = [ + this.frame('variable_name', token.start, token.end), + this.anon('=', next.start, next.end), + ]; + const value = this.parseExpression(st, 0); + if (value === null) this.hasError = true; + else kids.push(value); + return this.frame('variable_assignment', token.start, this.endOf(kids, next.end), kids); + } + return this.frame('word', token.start, token.end); + } + return this.frame('variable_name', token.start, token.end); + } + case 'word': { + this.exprNext(st); + // In a test command a `-digits` operand is unary minus, not a + // number literal (`[[ $a == -1 ]]` → unary(-, number 1) — the + // opposite of the max-length rule in ${v: -1}). + if (st.mode === 'test' && /^-\d+$/.test(token.text)) { + return this.frame('unary_expression', token.start, token.end, [ + this.anon('-', token.start, token.start + 1), + this.frame('number', token.start + 1, token.end), + ]); + } + return this.parseLiteral(token.start, token.end); + } + case 'subst': + case 'string': + this.exprNext(st); + return token.frame!; + case 'lparen': { + this.exprNext(st); + if (this.exprDepth >= MAX_PARSE_DEPTH) { + this.hasError = true; + st.pos = st.end; + st.lookahead = null; + return this.frame('ERROR', token.start, st.end); + } + this.exprDepth++; + st.parenDepth++; + const kids: Frame[] = [this.anon('(', token.start, token.end)]; + const inner = this.parseExpression(st, 0); + if (inner !== null) kids.push(inner); + if (st.mode === 'c') { + // _c_parenthesized_expression: comma-separated expressions. + for (;;) { + const comma = this.exprPeek(st); + if (comma.kind !== 'op' || comma.text !== ',') break; + this.exprNext(st); + kids.push(this.anon(',', comma.start, comma.end)); + const next = this.parseExpression(st, 0); + if (next === null) { + this.hasError = true; + break; + } + kids.push(next); + } + } + let end = this.endOf(kids, token.end); + const close = this.exprPeek(st); + if (close.kind === 'rparen') { + this.exprNext(st); + kids.push(this.anon(')', close.start, close.end)); + end = close.end; + } else { + this.hasError = true; // unterminated parenthesized expression + } + st.parenDepth--; + this.exprDepth--; + return this.frame('parenthesized_expression', token.start, end, kids); + } + case 'op': + case 'testop': + case 'unknown': { + // An operator where an operand was expected: recover. + this.exprNext(st); + this.hasError = true; + return this.frame('ERROR', token.start, token.end); + } + } + } + + /** The right side of `=~` in a test command, as a list of frames. Quote + * rules from the reference scanner: a regex STARTING with a single quote + * folds the quoted segments into the regex token (`' boop '(.*)$` is one + * regex) unless the whole thing is just a quoted string plus plain word + * characters (`'a'b` → raw_string + word); a quote appearing mid-run + * makes the whole thing a plain concatenation (`a'b'c`). Whitespace + * inside (…) stays inside the regex token. */ + private parseTestRegex(st: ExprState): Frame[] | null { + let i = st.pos; + while (i < st.end && (this.source[i] === ' ' || this.source[i] === '\t' || this.source[i] === '\r')) i++; + st.pos = i; + st.lookahead = null; + if (i >= st.end) return null; + const ch = this.source[i]!; + if (ch === '"') { + const [piece, next] = this.parseString(i, st.end); + st.pos = next; + return [piece]; + } + if (ch === '$') { + // An expansion holds the pattern: parse a normal operand. + const operand = this.parseExpression(st, PREC_TEST + 1); + return operand === null ? null : [operand]; + } + // Scan the run as one regex token: it ends at whitespace that is not + // protected by an open paren (the reference scanner includes whitespace + // inside (…) in the token), at an unbalanced closer, or at the region + // end. Single quotes toggle an in-quote state; `$`-constructs are raw + // text (the scanner does not split them out here). + let j = i; + let inQuote = false; + let hasQuote = false; + let parenDepth = 0; + let sinceTick = 0; + while (j < st.end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const c = this.source[j]!; + if (!inQuote && parenDepth === 0 && (c === ' ' || c === '\t' || c === '\r' || c === '\n')) break; + if (c === "'") { + hasQuote = true; + inQuote = !inQuote; + j++; + continue; + } + if (c === '\\') { + j += 2; + continue; + } + if (!inQuote) { + if (c === '(') parenDepth++; + else if (c === ')') { + if (parenDepth === 0) break; + parenDepth--; + } + } + j++; + } + st.pos = j; + if (ch === "'") { + // Quoted string plus plain word characters → a plain literal. + if (/^'[^']*'\w*$/.test(this.text(i, j))) return [this.parseLiteral(i, j)]; + return [this.frame('regex', i, j)]; + } + if (hasQuote) return [this.parseLiteral(i, j)]; + return [this.frame('regex', i, j)]; + } + + /** + * Pattern-shaped right side of ==/!= in a test command (the grammar's + * _extglob_blob). Handles what a plain expression parse cannot: + * - a whole extglob group pattern (+(a|b), X/@(default).vim with X a + * glob, *([0-9])([0-9])) as ONE extglob_pattern, when the scanner's + * first/second character rules accept it; + * - glob text mixed with one quote/expansion (*${var2}*, *"7f 4c"*), + * split into extglob_pattern pieces around the construct. + * Returns null (no state touched) for plain operands — those go through + * the normal expression parse plus convertTestRightSide. + */ + private tryParseTestPattern(st: ExprState): { frames: Frame[]; end: number } | null { + let i = st.pos; + while (i < st.end && (this.source[i] === ' ' || this.source[i] === '\t' || this.source[i] === '\r')) i++; + if (i >= st.end) return null; + // Find the end of the pattern: unquoted whitespace at paren depth 0. + let depth = 0; + let j = i; + let sawGroup = false; + let sawConstruct = false; + while (j < st.end) { + const ch = this.source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + sawConstruct = true; + j = skipDoubleQuoted(this.source, this.budget, j, st.end); + continue; + } + if (ch === "'") { + sawConstruct = true; + j = skipSingleQuoted(this.source, this.budget, j, st.end); + continue; + } + if (ch === '`') { + sawConstruct = true; + j = skipBacktick(this.source, this.budget, j, st.end); + continue; + } + if (ch === '$') { + sawConstruct = true; + j = skipDollar(this.source, this.budget, j, st.end); + continue; + } + if ((ch === '?' || ch === '*' || ch === '+' || ch === '@' || ch === '!') && this.source[j + 1] === '(') { + sawGroup = true; + } + if (ch === '(') { + depth++; + j++; + continue; + } + if (ch === ')') { + if (depth > 0) { + depth--; + j++; + continue; + } + break; + } + if (depth === 0 && (ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n')) break; + j++; + } + const hasGroup = sawGroup; + const hasConstruct = sawConstruct; + // A pattern containing an extglob group but no construct: one + // extglob_pattern node when the scanner accepts it; rejected groups + // keep the normal (error) path. + if (hasGroup && !hasConstruct) { + if (!this.extglobGroupAccepted(i, j)) return null; + st.pos = j; + st.lookahead = null; + return { frames: [this.frame('extglob_pattern', i, j)], end: j }; + } + if (!hasConstruct) return null; + // Glob text (possibly with a group) mixed with one quote/expansion: + // extglob_pattern pieces around the construct (`@(*${x}.tar|*.sig)` → + // extglob "@(*", expansion, extglob ".tar|*.sig)"). + const pieces = this.parseExtglobBlob(i, j, hasGroup ? (s, e) => this.extglobGroupAccepted(s, e) : undefined); + if (pieces === null) return null; + st.pos = j; + st.lookahead = null; + return { frames: pieces, end: j }; + } + + /** Reference-shaped right-hand sides for test comparisons: + * - after `=`, a bare word is a regex when it contains a glob character + * or `=` — but only outside parentheses (`[[ a = b*c ]]` → regex, + * `[[ a = b=c ]]` → regex, `[[ ( $a = x* ) ]]` → word); + * - after `==`/`!=`, a bare word follows the scanner.c extglob_pattern + * rule — see isTestExtglob. + * The right side may arrive as a concatenation of word/number fragments + * (the lexer splits [ ] into their own pieces, so `[0-9]*` is not one + * word); bare fragment runs are judged — and converted — as a whole. */ + private convertTestRightSide(operator: string, right: Frame, st: ExprState): Frame { + if (right.type !== 'word' && !(right.type === 'concatenation' && this.isBarePieces(right))) { + return right; + } + const text = this.text(right.start, right.end); + if (operator === '=' && st.parenDepth === 0 && /[*?[\]=]/.test(text)) { + return this.frame('regex', right.start, right.end); + } + if ((operator === '==' || operator === '!=') && this.isTestExtglob(right)) { + return this.frame('extglob_pattern', right.start, right.end); + } + return right; + } + + /** True when a concatenation consists only of word/number fragments, so + * it can be treated as one bare word in test comparisons. */ + private isBarePieces(frame: Frame): boolean { + return frame.children.every((child) => child.type === 'word' || child.type === 'number'); + } + + /** + * The extglob_pattern scan rule from tree-sitter-bash's scanner.c + * (the `extglob_pattern:` label), reduced to what a bare test word can + * reach (test words never contain blanks, quotes, `$`, `|` or parens): + * - the first character must be a letter or one of `?*+@!-)\.[`; + * - a leading `-letter` is just a word (`[[ $a == -foo ]]` → word); + * - a single character becomes a glob only when directly followed by + * whitespace (`[[ $a == b ]]` → glob, `[[ (a == b) ]]` → word); + * - a `-letters` second run followed by `)`, `\` or `.` stays a word; + * - otherwise the second character must be alphanumeric or one of + * `[?/\_*` (skipped when the word starts with `[`), and the word is + * a glob iff any character is neither a letter nor `.` — escapes + * count only the character after the backslash when it is not a + * space/quote (`foo\ bar` → word, `b\*c` → glob). + */ + private isTestExtglob(right: Frame): boolean { + const text = this.text(right.start, right.end); + const nextChar = this.source[right.end]; + const isSpace = (ch: string | undefined): boolean => ch === ' ' || ch === '\t' || ch === '\r'; + const isAlpha = (ch: string): boolean => /[A-Za-z]/.test(ch); + const isAlnum = (ch: string): boolean => /[A-Za-z0-9]/.test(ch); + const c0 = text[0]!; + if (!isAlpha(c0) && !'?*+@!-)\\.['.includes(c0)) return false; + let sawNonAlphaDot = !isAlpha(c0); + let i = 1; + if (i >= text.length) { + if (isSpace(nextChar) || nextChar === '|') return true; + if ((c0 === ')' || c0 === '*') && nextChar === ')' && isSpace(this.source[right.end + 1])) { + return sawNonAlphaDot; + } + return false; + } + // "-\w" is just a word after ==/!= in the reference. + if (c0 === '-' && isAlpha(text[i]!)) return false; + if (text[i] === '-') { + // "-\w" in second position: just a word, unless something special + // follows the run. + i++; + while (i < text.length && isAlnum(text[i]!)) i++; + const after = i < text.length ? text[i]! : nextChar; + if (after === ')' || after === '\\' || after === '.') return false; + if (i >= text.length) return true; + } + // The second-character check is skipped for a leading `[` (the scanner + // does not advance past it before checking). + if (c0 !== '[' && !isAlnum(text[i]!) && !'[?/\\_*'.includes(text[i]!)) return false; + for (; i < text.length; i++) { + const ch = text[i]!; + if (ch === '\\') { + // A backslash never counts itself; an escaped space/quote is + // consumed silently (anything else is judged on its own). + const after = text[i + 1]; + if (after === ' ' || after === '\t' || after === '\r' || after === '"') i++; + continue; + } + if (!isAlpha(ch) && ch !== '.') sawNonAlphaDot = true; + } + return sawNonAlphaDot; + } + + /** Character-level tokenizer for the expression engine over [st.pos, + * st.end). Produces one token per call and advances st.pos. */ + private scanExprToken(st: ExprState): ExprToken { + const end = st.end; + let i = st.pos; + let sinceTick = 0; + for (;;) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + if (i >= end) break; + const ch = this.source[i]!; + if (ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n') { + i++; + continue; + } + if (ch === '\\' && this.source[i + 1] === '\n') { + i += 2; // line continuation + continue; + } + break; + } + st.pos = i; + if (i >= end) return { kind: 'end', start: i, end: i, text: '' }; + const ch = this.source[i]!; + if (st.mode === 'test') return this.scanTestToken(st, i, ch); + + // Arithmetic / c-style-for tokens. + if (ch >= '0' && ch <= '9') { + let j = i; + if (ch === '0' && (this.source[i + 1] === 'x' || this.source[i + 1] === 'X')) { + j = i + 2; + while (j < end && /[0-9a-fA-F]/.test(this.source[j]!)) j++; + } else { + while (j < end && this.source[j]! >= '0' && this.source[j]! <= '9') j++; + if (this.source[j] === '#') { + j++; + while (j < end && /[0-9A-Za-z@_]/.test(this.source[j]!)) j++; + } + } + st.pos = j; + return { kind: 'number', start: i, end: j, text: this.text(i, j) }; + } + if (/[A-Za-z_]/.test(ch)) { + let j = i + 1; + while (j < end && /\w/.test(this.source[j]!)) j++; + if (this.source[j] === '[' && j < end) { + // Subscript: name[index] — the index is a literal. + const sub = this.scanBalanced(j, end, '[', ']'); + if (!sub.balanced) this.hasError = true; + const indexEnd = sub.balanced ? sub.end - 1 : sub.end; + const kids: Frame[] = [this.frame('variable_name', i, j), this.anon('[', j, j + 1)]; + if (indexEnd > j + 1) { + kids.push(this.parseLiteral(j + 1, indexEnd)); + } else { + this.hasError = true; // empty subscript index + } + if (sub.balanced) kids.push(this.anon(']', sub.end - 1, sub.end)); + st.pos = sub.end; + return { kind: 'subst', start: i, end: sub.end, text: this.text(i, sub.end), frame: this.frame('subscript', i, sub.end, kids) }; + } + st.pos = j; + return { kind: 'ident', start: i, end: j, text: this.text(i, j) }; + } + if (ch === '$') { + const dollar = this.parseDollar(i, end); + if (dollar !== null) { + st.pos = dollar[1]; + return { kind: 'subst', start: i, end: dollar[1], text: this.text(i, dollar[1]), frame: dollar[0] }; + } + this.hasError = true; // bare $ inside arithmetic + st.pos = i + 1; + return { kind: 'subst', start: i, end: i + 1, text: '$', frame: this.frame('ERROR', i, i + 1) }; + } + if (ch === '"') { + const [piece, next] = this.parseString(i, end); + st.pos = next; + return { kind: 'string', start: i, end: next, text: this.text(i, next), frame: piece }; + } + if (ch === '`') { + const [piece, next] = this.parseBacktickSubstitution(i, end); + st.pos = next; + return { kind: 'subst', start: i, end: next, text: this.text(i, next), frame: piece }; + } + if (ch === '(') { + st.pos = i + 1; + return { kind: 'lparen', start: i, end: i + 1, text: '(' }; + } + if (ch === ')') { + st.pos = i + 1; + return { kind: 'rparen', start: i, end: i + 1, text: ')' }; + } + for (const operator of EXPRESSION_OPERATORS) { + if (i + operator.length <= end && this.source.startsWith(operator, i)) { + st.pos = i + operator.length; + return { kind: 'op', start: i, end: st.pos, text: operator }; + } + } + // Unknown character: recover one char at a time. + st.pos = i + 1; + return { kind: 'unknown', start: i, end: i + 1, text: ch }; + } + + /** Tokenizer for test-command expressions ([[ … ]] / [ … ]). Words stop + * at whitespace and ( ) < > & | only — `=` and `!` are word characters + * unless they start a token (`a=b` is one word in the reference, while + * spaced `=`/`==`/`!=`/`=~` are operators). */ + private scanTestToken(st: ExprState, i: number, ch: string): ExprToken { + const end = st.end; + if (ch === '(') { + // `((…))` inside a test is an arithmetic_expansion when the balanced + // region closes with `))` (`[[ ((a)) == x ]]` in the reference); + // otherwise the parens nest as parenthesized_expressions. + if (this.source[i + 1] === '(') { + const scan = this.scanBalanced(i, end, '(', ')'); + if (scan.balanced && scan.end - 2 > i + 1 && this.source[scan.end - 2] === ')') { + const [piece, next] = this.parseParenArithmetic(i, end); + st.pos = next; + return { kind: 'subst', start: i, end: next, text: this.text(i, next), frame: piece }; + } + } + st.pos = i + 1; + return { kind: 'lparen', start: i, end: i + 1, text: '(' }; + } + if (ch === ')') { + st.pos = i + 1; + return { kind: 'rparen', start: i, end: i + 1, text: ')' }; + } + if (ch === '&' && this.source[i + 1] === '&') { + st.pos = i + 2; + return { kind: 'op', start: i, end: i + 2, text: '&&' }; + } + if (ch === '|' && this.source[i + 1] === '|') { + st.pos = i + 2; + return { kind: 'op', start: i, end: i + 2, text: '||' }; + } + if (ch === '<' || ch === '>') { + const wide = this.source[i + 1] === '=' ? 2 : 1; + st.pos = i + wide; + return { kind: 'op', start: i, end: i + wide, text: this.text(i, i + wide) }; + } + if (ch === '!' || ch === '=') { + if (st.expectOperator) { + // After an operand these are comparison operators even when the + // next word is attached (`=~^x`, `==b`, `=b` all split). + const two = this.source.slice(i, i + 2); + const wide = two === '==' || two === '=~' || two === '!=' ? 2 : 1; + st.pos = i + wide; + return { kind: 'op', start: i, end: i + wide, text: this.text(i, i + wide) }; + } + // At operand position only a standalone `!` is the prefix operator; + // everything else (=b, !x, !=b) is word material in the reference. + const after = this.source[i + 1]; + if (ch === '!' && (after === undefined || after === ' ' || after === '\t' || after === '\r' || i + 1 >= end)) { + st.pos = i + 1; + return { kind: 'op', start: i, end: i + 1, text: '!' }; + } + // Fall through to the word region below. + } + if (ch === '-' && /[A-Za-z]/.test(this.source[i + 1] ?? '')) { + // test_operator: -letters followed by whitespace AND by a real + // operand (scanner.c's rule: -\w followed by an operator, the closer + // or nothing is just a word — `[[ -foo == x ]]`, `[[ $a == -foo ]]`, + // `[[ -f ]]` all keep `-foo`/`-f` as words in the reference). + let j = i + 1; + while (j < end && /[A-Za-z]/.test(this.source[j]!)) j++; + const after = this.source[j]; + if (j < end && (after === ' ' || after === '\t' || after === '\r')) { + let k = j; + while (k < end && (this.source[k] === ' ' || this.source[k] === '\t' || this.source[k] === '\r')) k++; + const next = this.source[k]; + if (k < end && next !== '=' && next !== ']' && next !== '&' && next !== '|' && next !== ')') { + st.pos = j; + return { kind: 'testop', start: i, end: j, text: this.text(i, j) }; + } + } + } + // A word region: quote- and substitution-aware, stopping at blanks and + // the operator characters above. + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const c = this.source[j]!; + if (c === ' ' || c === '\t' || c === '\r' || c === '\n') break; + if (c === '(' || c === ')' || c === '<' || c === '>' || c === '&' || c === '|') break; + if (c === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (c === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (c === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (c === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if (c === '\\') { + j += 2; + continue; + } + j++; + } + if (j === i) j++; // defensive: never stall + st.pos = j; + // A lone arithmetic operator character is an operator, not a word. + if (j === i + 1 && (ch === '+' || ch === '*' || ch === '/' || ch === '%')) { + return { kind: 'op', start: i, end: j, text: ch }; + } + return { kind: 'word', start: i, end: j, text: this.text(i, j) }; + } + + // ------------------------------------------------------------ test command + + /** + * test_command in the `(( expression ))` form — only reached for + * statement-position `((word++…))` / `((word--…))` (see PAREN_TEST_RE); + * every other `((…))` is an arithmetic_expansion. The header arrived as + * one word token (see the lexer), so no repositioning is needed. + */ + private parseParenTestCommand(): Frame { + const token = this.lexer.next(); + const closed = + token.end - token.start >= 4 && this.source[token.end - 2] === ')' && this.source[token.end - 1] === ')'; + if (!closed) this.hasError = true; + const kids: Frame[] = [this.anon('((', token.start, token.start + 2)]; + const innerEnd = closed ? token.end - 2 : token.end; + if (innerEnd > token.start + 2) { + const st = this.newExprState(token.start + 2, innerEnd, 'test'); + const expression = this.parseExpression(st, 0); + if (expression !== null) kids.push(expression); + const leftover = this.exprLeftover(st); + if (leftover !== null) kids.push(leftover); + } + if (closed) { + kids.push(this.anon('))', token.end - 2, token.end)); + } + return this.frame('test_command', token.start, this.endOf(kids, token.end), kids); + } + + /** + * test_command: `[[ expression ]]` or `[ expression ]`. The peeked token + * is the single-character word `[`; `[[` is detected by the adjacent + * second `[` in the source. The expression range is scanned + * character-wise (quote/substitution aware) up to the closer or the end + * of the line, parsed by the expression engine in test mode, and the main + * lexer is then repositioned past the closer. + */ + private parseTestCommand(): Frame { + const openToken = this.lexer.next(); + const double = this.source[openToken.end] === '['; + const opener = double ? '[[' : '['; + const closer = double ? ']]' : ']'; + const exprStart = openToken.end + (double ? 1 : 0); + const kids: Frame[] = [this.anon(opener, openToken.start, exprStart)]; + const scan = this.scanTestCloser(exprStart, double); + // A single-bracket test whose body contains a top-level redirect is a + // redirected statement in the reference, not a test expression + // (`[ ! command -v go &>/dev/null ]` → redirected_statement with a + // negated_command inside the test_command). + if (!double && scan.closerStart > exprStart && this.rangeHasTopLevelRedirect(exprStart, scan.closerStart)) { + const statements = this.parseScopedStatements(exprStart, scan.closerStart); + kids.push(...statements); + let end = scan.closerStart; + if (scan.found) { + kids.push(this.anon(closer, scan.closerStart, scan.afterCloser)); + end = scan.afterCloser; + } else { + this.hasError = true; + } + this.lexer.reposition(end); + return this.frame('test_command', openToken.start, end, kids); + } + if (scan.closerStart > exprStart) { + const st = this.newExprState(exprStart, scan.closerStart, 'test'); + const expression = this.parseExpression(st, 0); + if (expression !== null) kids.push(expression); + const leftover = this.exprLeftover(st); + if (leftover !== null) kids.push(leftover); + } + let end = scan.closerStart; + if (scan.found) { + if (double && kids.length === 1) { + // `[[ ]]` (empty expression): the reference parses the closer text + // as two word pieces inside a concatenation and inserts a + // zero-width `]]` after them. + this.hasError = true; + kids.push( + this.frame('concatenation', scan.closerStart, scan.afterCloser, [ + this.frame('word', scan.closerStart, scan.closerStart + 1), + this.frame('word', scan.closerStart + 1, scan.afterCloser), + ]), + ); + kids.push(this.anon(closer, scan.afterCloser, scan.afterCloser)); + } else { + kids.push(this.anon(closer, scan.closerStart, scan.afterCloser)); + } + end = scan.afterCloser; + } else { + // Unterminated: keep a zero-width closer and flag the error (our + // documented recovery policy); trailing blanks stay outside the node, + // as in the reference. The reference inserts a zero-width closer only + // for an unterminated `[` or for `[[` ending in a complete unary + // test — other unterminated `[[` forms become an ERROR node there. + this.hasError = true; + let closeAt = scan.closerStart; + while (closeAt > exprStart && (this.source[closeAt - 1] === ' ' || this.source[closeAt - 1] === '\t' || this.source[closeAt - 1] === '\r')) { + closeAt--; + } + kids.push(this.anon(closer, closeAt, closeAt)); + end = closeAt; + } + this.lexer.reposition(end); + return this.frame('test_command', openToken.start, end, kids); + } + + /** Whether [start, end) contains a top-level redirect operator (`<`/`>` + * outside quotes, substitutions and `<(`/`>(`). Used to recognize the + * redirected-statement form of a single-bracket test command. */ + private rangeHasTopLevelRedirect(start: number, end: number): boolean { + let j = start; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (ch === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if ((ch === '<' || ch === '>') && this.source[j + 1] !== '(') return true; + j++; + } + return false; + } + + /** Find the closer of a test command, skipping quotes and substitutions. + * Bounded by the end of the line and the lexer's range. */ + private scanTestCloser(start: number, double: boolean): { closerStart: number; afterCloser: number; found: boolean } { + const end = this.lexer.rangeEnd; + let j = start; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\n') break; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (ch === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if ((ch === '<' || ch === '>') && this.source[j + 1] === '(') { + j = this.scanBalanced(j + 1, end, '(', ')').end; + continue; + } + if (ch === ']') { + if (double) { + if (this.source[j + 1] === ']') return { closerStart: j, afterCloser: j + 2, found: true }; + } else { + return { closerStart: j, afterCloser: j + 1, found: true }; + } + } + j++; + } + return { closerStart: j, afterCloser: j, found: false }; + } + + /** heredoc_body content: heredoc_content chunks interleaved with + * expansions and command substitutions (unquoted delimiters only). + * Unlike tree-sitter-bash's scanner, every plain chunk — including the + * leading one — becomes a heredoc_content node. */ + private parseHeredocContent(start: number, end: number): Frame[] { + const pieces: Frame[] = []; + let chunkStart = start; + let i = start; + const flush = (upto: number): void => { + if (upto > chunkStart) { + pieces.push(this.frame('heredoc_content', chunkStart, upto)); + } + }; + let sinceTick = 0; + while (i < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[i]!; + if (ch === '\\') { + i += 2; // escaped characters stay literal (and suppress expansion) + continue; + } + if (ch === '$') { + // `$'` is NOT an ANSI-C string inside a heredoc body — bash does not + // expand those there and the reference keeps the text as plain + // content too (except for its line-start quirk, see README). + if (this.source[i + 1] === "'") { + i++; + continue; + } + const dollar = this.parseDollar(i, end); + if (dollar !== null) { + flush(i); + pieces.push(dollar[0]); + i = dollar[1]; + chunkStart = i; + continue; + } + i++; + continue; + } + if (ch === '`') { + flush(i); + const [piece, next] = this.parseBacktickSubstitution(i, end); + pieces.push(piece); + i = next; + chunkStart = i; + continue; + } + i++; + } + flush(end); + return pieces; + } + + /** Balanced-scan wrapper returning whether the close was actually found. */ + private scanBalanced(i: number, end: number, open: string, close: string): BalancedScan { + return scanBalanced(this.source, this.budget, i, end, open, close); + } +} + +/** + * Convert a finished frame tree into SyntaxNodeBuilders. Iterative + * (explicit stack) so deep trees cannot overflow the call stack; node + * creation here is not budget-ticked because every frame already ticked the + * budget when it was created (one frame ↔ one node). + */ +export function materialize(root: Frame, source: string): SyntaxNodeBuilder { + const nodes = new Map(); + const order: Frame[] = []; + const stack: Frame[] = [root]; + while (stack.length > 0) { + const frame = stack.pop()!; + order.push(frame); + for (const child of frame.children) stack.push(child); + } + for (const frame of order) { + nodes.set( + frame, + new SyntaxNodeBuilder({ + type: frame.type, + source, + startIndex: frame.start, + endIndex: frame.end, + isNamed: frame.isNamed, + }), + ); + } + for (const frame of order) { + const node = nodes.get(frame)!; + for (const child of frame.children) { + node.addChild(nodes.get(child)!); + } + } + return nodes.get(root)!; +} diff --git a/packages/tree-sitter-bash/test/differential.test.ts b/packages/tree-sitter-bash/test/differential.test.ts new file mode 100644 index 0000000000..0c185e5125 --- /dev/null +++ b/packages/tree-sitter-bash/test/differential.test.ts @@ -0,0 +1,169 @@ +// test/differential.test.ts +// +// Differential suite: every sample in test/fixtures/differential/ and every +// official tree-sitter-bash 0.25.0 corpus case in test/fixtures/corpus/ is +// parsed by BOTH this package and the reference (web-tree-sitter + the +// official wasm build) and the normalized trees are compared byte-for-byte. +// +// - @match samples must produce a tree identical to the reference; +// - @known-diff samples must (a) still differ from the reference and +// (b) match the stored dump of our parser exactly — so a documented +// deviation cannot silently drift, and a fixed deviation fails loudly +// ("remove it from the known list"). +// +// The known-difference registry (test/helpers/known-differences.ts) is the +// single source of truth and is cross-checked against README.md's Known +// differences section here. + +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; + +import { beforeAll, describe, expect, it } from 'vitest'; + +import { compareSource, diffDumps, ourDump, parseCorpusFile, parseFixtureFile, referenceDump } from './helpers/differential'; +import type { FixtureSample } from './helpers/differential'; +import { KNOWN_DIFFERENCES, isKnownDifferenceId } from './helpers/known-differences'; + +const PKG_ROOT = path.resolve(import.meta.dirname, '..'); +const DIFF_DIR = path.join(PKG_ROOT, 'test/fixtures/differential'); +const CORPUS_DIR = path.join(PKG_ROOT, 'test/fixtures/corpus'); + +interface LocatedSample extends FixtureSample { + file: string; +} + +function loadDifferentialFixtures(): LocatedSample[] { + const out: LocatedSample[] = []; + for (const file of readdirSync(DIFF_DIR).filter((f) => f.endsWith('.txt')).toSorted()) { + const content = readFileSync(path.join(DIFF_DIR, file), 'utf8'); + for (const sample of parseFixtureFile(`${DIFF_DIR}/${file}`, content)) { + out.push({ ...sample, file }); + } + } + return out; +} + +function loadCorpusKnownDiffs(): Map { + const file = 'known-diffs.txt'; + const content = readFileSync(path.join(CORPUS_DIR, file), 'utf8'); + const map = new Map(); + for (const sample of parseFixtureFile(`${CORPUS_DIR}/${file}`, content)) { + // description: "corpus ::" + const key = sample.description.replace(/^corpus /, ''); + map.set(key, { ...sample, file }); + } + return map; +} + +function label(sample: LocatedSample): string { + return `${sample.file}:${sample.line} ${sample.kind === 'match' ? 'match' : `known-diff ${sample.id}`} — ${sample.description.slice(0, 60)}`; +} + +const fixtures = loadDifferentialFixtures(); +const corpusKnownDiffs = loadCorpusKnownDiffs(); +const corpusCases = readdirSync(CORPUS_DIR) + .filter((f) => f.endsWith('.txt') && f !== 'known-diffs.txt') + .toSorted() + .flatMap((file) => parseCorpusFile(readFileSync(path.join(CORPUS_DIR, file), 'utf8')).map((c) => ({ ...c, file }))); + +beforeAll(async () => { + // Fails with a descriptive error when the wasm reference cannot be loaded. + await compareSource('echo sanity'); +}); + +describe('differential fixtures', () => { + for (const sample of fixtures) registerFixtureTest(sample); +}); + +/** Register one fixture test (lookup table instead of conditionals — the + * vitest lint rules reject conditionals around `it`). */ +function registerFixtureTest(sample: LocatedSample): void { + const register: Record void> = { + match: () => { + it(label(sample), async () => { + const cmp = await compareSource(sample.source); + expect(cmp.diff).toBe(''); + }); + }, + 'known-diff': () => { + it(label(sample), async () => { + const [ours, reference] = [ourDump(sample.source), await referenceDump(sample.source)]; + expect( + ours !== reference, + 'this sample now matches the reference — remove it from the known-difference list (fixture + registry + README)', + ).toBe(true); + expect(ours === sample.expectedOurs, `deviation shape drifted:\n${diffDumps(ours, sample.expectedOurs!)}`).toBe( + true, + ); + }); + }, + }; + register[sample.kind](); +} + +describe('official tree-sitter-bash 0.25.0 corpus', () => { + for (const corpusCase of corpusCases) registerCorpusTest(corpusCase); +}); + +/** Register one corpus case test. */ +function registerCorpusTest(corpusCase: { file: string; name: string; input: string }): void { + const key = `${corpusCase.file}::${corpusCase.name}`; + const known = corpusKnownDiffs.get(key) ?? null; + const register: Record<'match' | 'known-diff', () => void> = { + match: () => { + it(`${key} (match)`, async () => { + const cmp = await compareSource(corpusCase.input); + expect(cmp.diff).toBe(''); + }); + }, + 'known-diff': () => { + it(`${key} (known-diff ${known!.id})`, async () => { + const [ours, reference] = [ourDump(corpusCase.input), await referenceDump(corpusCase.input)]; + expect( + ours !== reference, + 'this corpus case now matches the reference — remove it from known-diffs.txt, the registry and the README', + ).toBe(true); + expect(ours === known!.expectedOurs, `deviation shape drifted:\n${diffDumps(ours, known!.expectedOurs!)}`).toBe( + true, + ); + }); + }, + }; + register[known === null ? 'match' : 'known-diff'](); +} + +describe('known-difference registry consistency', () => { + const usedIds = new Set(); + for (const sample of fixtures) { + if (sample.id !== undefined) usedIds.add(sample.id); + } + for (const sample of corpusKnownDiffs.values()) { + if (sample.id !== undefined) usedIds.add(sample.id); + } + + it('every @known-diff id used in fixtures exists in the registry', () => { + for (const id of usedIds) { + expect(isKnownDifferenceId(id), `fixture uses unregistered id '${id}'`).toBe(true); + } + }); + + it('every registry id is exercised by at least one fixture or corpus case', () => { + for (const entry of KNOWN_DIFFERENCES) { + expect(usedIds.has(entry.id), `registry id '${entry.id}' has no fixture sample`).toBe(true); + } + }); + + it('every registry entry is anchored in README.md Known differences', () => { + const readme = readFileSync(path.join(PKG_ROOT, 'README.md'), 'utf8'); + const sectionMatch = /## Known differences[^\n]*\n([\s\S]*?)(?=\n## |\s*$)/.exec(readme); + expect(sectionMatch, 'README.md has no "## Known differences" section').not.toBeNull(); + // Whitespace-normalized so anchors may span line breaks in the README. + const section = sectionMatch![1]!.replaceAll(/\s+/g, ' '); + for (const entry of KNOWN_DIFFERENCES) { + expect( + section.includes(entry.readmeAnchor.replaceAll(/\s+/g, ' ')), + `README.md Known differences does not contain the anchor for '${entry.id}': ${entry.readmeAnchor}`, + ).toBe(true); + } + }); +}); diff --git a/packages/tree-sitter-bash/test/fixtures/README.md b/packages/tree-sitter-bash/test/fixtures/README.md new file mode 100644 index 0000000000..bcb4403b06 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/README.md @@ -0,0 +1,42 @@ +# Differential fixtures + +Fixtures for the differential test suite (`test/differential.test.ts`): +every sample is parsed by this package and by the reference tree-sitter-bash +0.25.0 parser (web-tree-sitter + wasm), and the normalized trees are +compared. + +## Format + +Blocks are separated by a line containing exactly `===`. Each block starts +with a directive line: + +``` +@match: + +=== +@known-diff : + +--- + +=== +``` + +- `@match`: the two trees must be identical. +- `@known-diff `: the sample documents a known deviation. `` must + exist in `test/helpers/known-differences.ts` (the registry is + cross-checked against the README's Known differences section). The block + must still deviate from the reference, and our tree must match the stored + dump exactly — so deviations cannot silently drift, and a fixed deviation + fails the test with a reminder to remove it from the list. + +`differential/` holds curated samples grouped by theme (statements, +redirects, heredoc, expansions, test-command, arithmetic, case, recovery). +`corpus/` holds the official tree-sitter-bash corpus (see its README); +corpus cases that deviate are listed in `corpus/known-diffs.txt` in the same +format. + +The normalized dump is a pre-order walk of every node (named and +anonymous): one line per node — `type [start,end] "text preview"`, +anonymous types in parentheses — preceded by a `hasError:` line. Offsets +are UTF-16 code units on both sides (web-tree-sitter reports UTF-16 +offsets for string input, same as this parser). diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/README.md b/packages/tree-sitter-bash/test/fixtures/corpus/README.md new file mode 100644 index 0000000000..12524df4ed --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/README.md @@ -0,0 +1,21 @@ +# Official tree-sitter-bash corpus + +The five `*.txt` files in this directory are an unmodified copy of the +official tree-sitter-bash test corpus: + +- Source: https://github.com/tree-sitter/tree-sitter-bash/tree/v0.25.0/test/corpus +- Version: tag `v0.25.0` +- Retrieved: 2026-07-21 (via the GitHub release tarball; only `test/corpus/` + was taken) + +They are inputs to `test/differential.test.ts`: every corpus case is parsed +by both this package and the live reference parser (web-tree-sitter + +tree-sitter-bash.wasm v0.25.0) and the trees are compared byte-for-byte. +The expected S-expressions inside the corpus files are not used — the live +wasm build is the comparison target. + +`known-diffs.txt` lists the corpus cases whose trees deliberately deviate +(see the "Known differences" section of the package README), in the fixture +format described in `test/fixtures/README.md`. A corpus case listed there +must still deviate from the reference AND keep the exact stored deviation +shape; every other corpus case must match the reference byte-for-byte. diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/commands.txt b/packages/tree-sitter-bash/test/fixtures/corpus/commands.txt new file mode 100644 index 0000000000..63e95ce2d9 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/commands.txt @@ -0,0 +1,725 @@ +=============================== +Commands +=============================== + +whoami + +--- + +(program + (command (command_name (word)))) + +=============================== +Commands with arguments +=============================== + +cat file1.txt +git diff --word-diff=color -- file1.txt file2.txt +echo $sing\ +levar + +--- + +(program + (command (command_name (word)) (word)) + (command (command_name (word)) (word) (word) (word) (word) (word)) + (command (command_name (word)) (simple_expansion (variable_name)) (word))) + +=============================== +Quoted command names +=============================== + +"$a/$b" c + +--- + +(program + (command + (command_name (string (simple_expansion (variable_name)) (string_content) (simple_expansion (variable_name)))) + (word))) + +=============================== +Commands with numeric arguments +=============================== + +exit 1 + +--- + +(program + (command (command_name (word)) (number))) + +=================================== +Commands with environment variables +=================================== + +VAR1=1 ./script/test +VAR1=a VAR2="ok" git diff --word-diff=color + +--- + +(program + (command + (variable_assignment (variable_name) (number)) + (command_name (word))) + (command + (variable_assignment (variable_name) (word)) + (variable_assignment (variable_name) (string (string_content))) + (command_name (word)) + (word) + (word))) + +=================================== +Empty environment variables +=================================== + +VAR1= +VAR2= echo + +--- + +(program + (variable_assignment (variable_name)) + (command (variable_assignment (variable_name)) (command_name (word)))) + +=============================== +File redirects +=============================== + +whoami > /dev/null +cat a b > /dev/null +2>&1 whoami +echo "foobar" >&2 +[ ! command -v go &>/dev/null ] && return + +if [ ]; then + >aa >bb +fi + +exec {VIRTWL[0]} {VIRTWL[1]} <&- >&- +exec {VIRTWL[0]}<&- {VIRTWL[1]}>&- + +grep 2>/dev/null -q "^/usr/bin/scponly$" /etc/shells + +x | /dev/null +cat a b >| /dev/null + +--- + +(program + (redirected_statement + (command (command_name (word))) + (file_redirect (word))) + (redirected_statement + (command (command_name (word)) (word) (word)) + (file_redirect (word)))) + +=============================== +Heredoc redirects +=============================== + +node < $tmpfile +a $B ${C} +EOF + +wc -l $tmpfile + +--- + +(program + (redirected_statement + (command + (command_name + (word))) + (heredoc_redirect + (heredoc_start) + (file_redirect + (simple_expansion + (variable_name))) + (heredoc_body + (simple_expansion + (variable_name)) + (heredoc_content) + (expansion + (variable_name)) + (heredoc_content)) + (heredoc_end))) + (command + (command_name + (word)) + (word) + (simple_expansion + (variable_name)))) + +================================= +Heredocs with many file redirects +================================= + +FOO=bar echo < err.txt > hello.txt +hello +EOF + +--- + +(program + (redirected_statement + body: (command + (variable_assignment + name: (variable_name) + value: (word)) + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + redirect: (file_redirect + descriptor: (file_descriptor) + destination: (word)) + redirect: (file_redirect + destination: (word)) + (heredoc_body) + (heredoc_end)))) + +================================= +Heredocs with pipes +================================= + +one < temp +EOF + +--- + +(program + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + (heredoc_body) + (heredoc_end))) + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + (heredoc_body) + (heredoc_end))) + (function_definition + name: (word) + body: (compound_statement + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + (heredoc_body) + (heredoc_end))))) + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + redirect: (file_redirect + destination: (word)) + (heredoc_body) + (heredoc_end)))) + +========================================== +Heredocs with weird characters +========================================== + +node <<_DELIMITER_WITH_UNDERSCORES_ +Hello. +_DELIMITER_WITH_UNDERSCORES_ + +node <<'```' +Hello. +``` + +node < /dev/null; + +<< $tmpfile +a $B ${C} +EOF + +wc -l $tmpfile +--- +hasError: false +program [0,50] "cat < $tmpfile\na $B ${C}\nEOF\n\nw..." + redirected_statement [0,34] "cat < $tmpfile\na $B ${C}\nEOF" + command [0,3] "cat" + command_name [0,3] "cat" + word [0,3] "cat" + heredoc_redirect [4,34] "< $tmpfile\na $B ${C}\nEOF" + (<<) [4,6] "<<" + heredoc_start [6,9] "EOF" + file_redirect [10,20] "> $tmpfile" + (>) [10,11] ">" + simple_expansion [12,20] "$tmpfile" + ($) [12,13] "$" + variable_name [13,20] "tmpfile" + heredoc_body [21,31] "a $B ${C}\n" + heredoc_content [21,23] "a " + simple_expansion [23,25] "$B" + ($) [23,24] "$" + variable_name [24,25] "B" + heredoc_content [25,26] " " + expansion [26,30] "${C}" + (${) [26,28] "${" + variable_name [28,29] "C" + (}) [29,30] "}" + heredoc_content [30,31] "\n" + heredoc_end [31,34] "EOF" + command [36,50] "wc -l $tmpfile" + command_name [36,38] "wc" + word [36,38] "wc" + word [39,41] "-l" + simple_expansion [42,50] "$tmpfile" + ($) [42,43] "$" + variable_name [43,50] "tmpfile" +=== +=== +@known-diff heredoc-content-chunks: corpus commands.txt::Heredocs with many file redirects +FOO=bar echo < err.txt > hello.txt +hello +EOF +--- +hasError: false +program [0,51] "FOO=bar echo < err.txt > hello..." + redirected_statement [0,51] "FOO=bar echo < err.txt > hello..." + command [0,12] "FOO=bar echo" + variable_assignment [0,7] "FOO=bar" + variable_name [0,3] "FOO" + (=) [3,4] "=" + word [4,7] "bar" + command_name [8,12] "echo" + word [8,12] "echo" + heredoc_redirect [13,51] "< err.txt > hello.txt\nhello\nEOF" + (<<) [13,15] "<<" + heredoc_start [15,18] "EOF" + file_redirect [19,29] "2> err.txt" + file_descriptor [19,20] "2" + (>) [20,21] ">" + word [22,29] "err.txt" + file_redirect [30,41] "> hello.txt" + (>) [30,31] ">" + word [32,41] "hello.txt" + heredoc_body [42,48] "hello\n" + heredoc_content [42,48] "hello\n" + heredoc_end [48,51] "EOF" +=== +=== +@known-diff heredoc-content-chunks: corpus commands.txt::Heredocs with pipes +one <&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done + +echo + +( for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done ) +--- +hasError: false +program [0,542] "for (( c=1; c<=5; c++ ))\ndo\n echo $c..." + c_style_for_statement [0,42] "for (( c=1; c<=5; c++ ))\ndo\n echo $c..." + (for) [0,3] "for" + ((() [4,6] "((" + variable_assignment [7,10] "c=1" + variable_name [7,8] "c" + (=) [8,9] "=" + number [9,10] "1" + (;) [10,11] ";" + binary_expression [12,16] "c<=5" + word [12,13] "c" + (<=) [13,15] "<=" + number [15,16] "5" + (;) [16,17] ";" + postfix_expression [18,21] "c++" + word [18,19] "c" + (++) [19,21] "++" + ())) [22,24] "))" + do_group [25,42] "do\n echo $c\ndone" + (do) [25,27] "do" + command [30,37] "echo $c" + command_name [30,34] "echo" + word [30,34] "echo" + simple_expansion [35,37] "$c" + ($) [35,36] "$" + variable_name [36,37] "c" + (done) [38,42] "done" + c_style_for_statement [44,81] "for (( c=1; c<=5; c++ )) {\n\techo $c\n}" + (for) [44,47] "for" + ((() [48,50] "((" + variable_assignment [51,54] "c=1" + variable_name [51,52] "c" + (=) [52,53] "=" + number [53,54] "1" + (;) [54,55] ";" + binary_expression [56,60] "c<=5" + word [56,57] "c" + (<=) [57,59] "<=" + number [59,60] "5" + (;) [60,61] ";" + postfix_expression [62,65] "c++" + word [62,63] "c" + (++) [63,65] "++" + ())) [66,68] "))" + compound_statement [69,81] "{\n\techo $c\n}" + ({) [69,70] "{" + command [72,79] "echo $c" + command_name [72,76] "echo" + word [72,76] "echo" + simple_expansion [77,79] "$c" + ($) [77,78] "$" + variable_name [78,79] "c" + (}) [80,81] "}" + c_style_for_statement [83,121] "for (( ; ; ))\ndo\n echo 'forever'\ndone" + (for) [83,86] "for" + ((() [87,89] "((" + (;) [90,91] ";" + (;) [92,93] ";" + ())) [94,96] "))" + do_group [97,121] "do\n echo 'forever'\ndone" + (do) [97,99] "do" + command [102,116] "echo 'forever'" + command_name [102,106] "echo" + word [102,106] "echo" + raw_string [107,116] "'forever'" + (done) [117,121] "done" + c_style_for_statement [123,190] "for ((cx = 0; c = $cx / $pf, c < $wc ..." + (for) [123,126] "for" + ((() [127,129] "((" + variable_assignment [129,135] "cx = 0" + variable_name [129,131] "cx" + (=) [132,133] "=" + number [134,135] "0" + (;) [135,136] ";" + variable_assignment [137,150] "c = $cx / $pf" + variable_name [137,138] "c" + (=) [139,140] "=" + binary_expression [141,150] "$cx / $pf" + simple_expansion [141,144] "$cx" + ($) [141,142] "$" + variable_name [142,144] "cx" + (/) [145,146] "/" + simple_expansion [147,150] "$pf" + ($) [147,148] "$" + variable_name [148,150] "pf" + (,) [150,151] "," + binary_expression [152,164] "c < $wc - $k" + word [152,153] "c" + (<) [154,155] "<" + binary_expression [156,164] "$wc - $k" + simple_expansion [156,159] "$wc" + ($) [156,157] "$" + variable_name [157,159] "wc" + (-) [160,161] "-" + simple_expansion [162,164] "$k" + ($) [162,163] "$" + variable_name [163,164] "k" + (;) [164,165] ";" + ())) [166,168] "))" + (;) [168,169] ";" + do_group [170,190] "do\n\t\techo \"$cx\"\ndone" + (do) [170,172] "do" + command [175,185] "echo \"$cx\"" + command_name [175,179] "echo" + word [175,179] "echo" + string [180,185] "\"$cx\"" + (") [180,181] "\"" + simple_expansion [181,184] "$cx" + ($) [181,182] "$" + variable_name [182,184] "cx" + (") [184,185] "\"" + (done) [186,190] "done" + c_style_for_statement [192,264] "for (( i = 4;;i--)) ; do echo $i; if ..." + (for) [192,195] "for" + ((() [196,198] "((" + variable_assignment [199,204] "i = 4" + variable_name [199,200] "i" + (=) [201,202] "=" + number [203,204] "4" + (;) [204,205] ";" + (;) [205,206] ";" + postfix_expression [206,209] "i--" + word [206,207] "i" + (--) [207,209] "--" + ())) [209,211] "))" + (;) [212,213] ";" + do_group [214,264] "do echo $i; if (( $i == 0 )); then br..." + (do) [214,216] "do" + command [217,224] "echo $i" + command_name [217,221] "echo" + word [217,221] "echo" + simple_expansion [222,224] "$i" + ($) [222,223] "$" + variable_name [223,224] "i" + (;) [224,225] ";" + if_statement [226,258] "if (( $i == 0 )); then break; fi" + (if) [226,228] "if" + command [229,242] "(( $i == 0 ))" + command_name [229,242] "(( $i == 0 ))" + arithmetic_expansion [229,242] "(( $i == 0 ))" + ((() [229,231] "((" + binary_expression [232,239] "$i == 0" + simple_expansion [232,234] "$i" + ($) [232,233] "$" + variable_name [233,234] "i" + (==) [235,237] "==" + number [238,239] "0" + ())) [240,242] "))" + (;) [242,243] ";" + (then) [244,248] "then" + command [249,254] "break" + command_name [249,254] "break" + word [249,254] "break" + (;) [254,255] ";" + (fi) [256,258] "fi" + (;) [258,259] ";" + (done) [260,264] "done" + comment [266,287] "# added post-bash-4.2" + c_style_for_statement [288,409] "for (( i = j = k = 1; i % 9 || (j *= ..." + (for) [288,291] "for" + ((() [292,294] "((" + variable_assignment [295,308] "i = j = k = 1" + variable_name [295,296] "i" + (=) [297,298] "=" + variable_assignment [299,308] "j = k = 1" + variable_name [299,300] "j" + (=) [301,302] "=" + variable_assignment [303,308] "k = 1" + variable_name [303,304] "k" + (=) [305,306] "=" + number [307,308] "1" + (;) [308,309] ";" + binary_expression [310,378] "i % 9 || (j *= -1, $( ((i%9)) || prin..." + binary_expression [310,315] "i % 9" + word [310,311] "i" + (%) [312,313] "%" + number [314,315] "9" + (||) [316,318] "||" + parenthesized_expression [319,378] "(j *= -1, $( ((i%9)) || printf \" \" >&..." + (() [319,320] "(" + binary_expression [320,327] "j *= -1" + word [320,321] "j" + (*=) [322,324] "*=" + unary_expression [325,327] "-1" + (-) [325,326] "-" + number [326,327] "1" + (,) [327,328] "," + command_substitution [329,366] "$( ((i%9)) || printf \" \" >&2; echo 0)" + ($() [329,331] "$(" + list [332,357] "((i%9)) || printf \" \" >&2" + command [332,339] "((i%9))" + command_name [332,339] "((i%9))" + arithmetic_expansion [332,339] "((i%9))" + ((() [332,334] "((" + binary_expression [334,337] "i%9" + variable_name [334,335] "i" + (%) [335,336] "%" + number [336,337] "9" + ())) [337,339] "))" + (||) [340,342] "||" + redirected_statement [343,357] "printf \" \" >&2" + command [343,353] "printf \" \"" + command_name [343,349] "printf" + word [343,349] "printf" + string [350,353] "\" \"" + (") [350,351] "\"" + string_content [351,352] " " + (") [352,353] "\"" + file_redirect [354,357] ">&2" + (>&) [354,356] ">&" + number [356,357] "2" + (;) [357,358] ";" + command [359,365] "echo 0" + command_name [359,363] "echo" + word [359,363] "echo" + number [364,365] "0" + ()) [365,366] ")" + (,) [366,367] "," + binary_expression [368,377] "k++ <= 10" + postfix_expression [368,371] "k++" + word [368,369] "k" + (++) [369,371] "++" + (<=) [372,374] "<=" + number [375,377] "10" + ()) [377,378] ")" + (;) [378,379] ";" + binary_expression [380,386] "i += j" + word [380,381] "i" + (+=) [382,384] "+=" + word [385,386] "j" + ())) [387,389] "))" + do_group [390,409] "do\nprintf \"$i\"\ndone" + (do) [390,392] "do" + command [393,404] "printf \"$i\"" + command_name [393,399] "printf" + word [393,399] "printf" + string [400,404] "\"$i\"" + (") [400,401] "\"" + simple_expansion [401,403] "$i" + ($) [401,402] "$" + variable_name [402,403] "i" + (") [403,404] "\"" + (done) [405,409] "done" + command [411,415] "echo" + command_name [411,415] "echo" + word [411,415] "echo" + subshell [417,542] "( for (( i = j = k = 1; i % 9 || (j *..." + (() [417,418] "(" + c_style_for_statement [419,540] "for (( i = j = k = 1; i % 9 || (j *= ..." + (for) [419,422] "for" + ((() [423,425] "((" + variable_assignment [426,439] "i = j = k = 1" + variable_name [426,427] "i" + (=) [428,429] "=" + variable_assignment [430,439] "j = k = 1" + variable_name [430,431] "j" + (=) [432,433] "=" + variable_assignment [434,439] "k = 1" + variable_name [434,435] "k" + (=) [436,437] "=" + number [438,439] "1" + (;) [439,440] ";" + binary_expression [441,509] "i % 9 || (j *= -1, $( ((i%9)) || prin..." + binary_expression [441,446] "i % 9" + word [441,442] "i" + (%) [443,444] "%" + number [445,446] "9" + (||) [447,449] "||" + parenthesized_expression [450,509] "(j *= -1, $( ((i%9)) || printf \" \" >&..." + (() [450,451] "(" + binary_expression [451,458] "j *= -1" + word [451,452] "j" + (*=) [453,455] "*=" + unary_expression [456,458] "-1" + (-) [456,457] "-" + number [457,458] "1" + (,) [458,459] "," + command_substitution [460,497] "$( ((i%9)) || printf \" \" >&2; echo 0)" + ($() [460,462] "$(" + list [463,488] "((i%9)) || printf \" \" >&2" + command [463,470] "((i%9))" + command_name [463,470] "((i%9))" + arithmetic_expansion [463,470] "((i%9))" + ((() [463,465] "((" + binary_expression [465,468] "i%9" + variable_name [465,466] "i" + (%) [466,467] "%" + number [467,468] "9" + ())) [468,470] "))" + (||) [471,473] "||" + redirected_statement [474,488] "printf \" \" >&2" + command [474,484] "printf \" \"" + command_name [474,480] "printf" + word [474,480] "printf" + string [481,484] "\" \"" + (") [481,482] "\"" + string_content [482,483] " " + (") [483,484] "\"" + file_redirect [485,488] ">&2" + (>&) [485,487] ">&" + number [487,488] "2" + (;) [488,489] ";" + command [490,496] "echo 0" + command_name [490,494] "echo" + word [490,494] "echo" + number [495,496] "0" + ()) [496,497] ")" + (,) [497,498] "," + binary_expression [499,508] "k++ <= 10" + postfix_expression [499,502] "k++" + word [499,500] "k" + (++) [500,502] "++" + (<=) [503,505] "<=" + number [506,508] "10" + ()) [508,509] ")" + (;) [509,510] ";" + binary_expression [511,517] "i += j" + word [511,512] "i" + (+=) [513,515] "+=" + word [516,517] "j" + ())) [518,520] "))" + do_group [521,540] "do\nprintf \"$i\"\ndone" + (do) [521,523] "do" + command [524,535] "printf \"$i\"" + command_name [524,530] "printf" + word [524,530] "printf" + string [531,535] "\"$i\"" + (") [531,532] "\"" + simple_expansion [532,534] "$i" + ($) [532,533] "$" + variable_name [533,534] "i" + (") [534,535] "\"" + (done) [536,540] "done" + ()) [541,542] ")" +=== +=== +@known-diff case-short-option-pattern: corpus statements.txt::Test command paren statefulness with a case glob +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] + +case ${out} in +*"not supported"*|\ +*"operation not supported"*) + ;; +esac + +case $1 in +-o) + owner=$2 + shift + ;; +-g) ;; +esac + +[[ a == \"+(?)\" ]] +--- +hasError: false +program [0,193] "[[ ${test} == @($(IFS='|'; echo \"${sk..." + test_command [0,49] "[[ ${test} == @($(IFS='|'; echo \"${sk..." + ([[) [0,2] "[[" + binary_expression [3,46] "${test} == @($(IFS='|'; echo \"${skip[..." + expansion [3,10] "${test}" + (${) [3,5] "${" + variable_name [5,9] "test" + (}) [9,10] "}" + (==) [11,13] "==" + extglob_pattern [14,16] "@(" + command_substitution [16,45] "$(IFS='|'; echo \"${skip[*]}\")" + ($() [16,18] "$(" + variable_assignment [18,25] "IFS='|'" + variable_name [18,21] "IFS" + (=) [21,22] "=" + raw_string [22,25] "'|'" + (;) [25,26] ";" + command [27,44] "echo \"${skip[*]}\"" + command_name [27,31] "echo" + word [27,31] "echo" + string [32,44] "\"${skip[*]}\"" + (") [32,33] "\"" + expansion [33,43] "${skip[*]}" + (${) [33,35] "${" + subscript [35,42] "skip[*]" + variable_name [35,39] "skip" + ([) [39,40] "[" + word [40,41] "*" + (]) [41,42] "]" + (}) [42,43] "}" + (") [43,44] "\"" + ()) [44,45] ")" + extglob_pattern [45,46] ")" + (]]) [47,49] "]]" + case_statement [51,123] "case ${out} in\n*\"not supported\"*|\\\n*\"..." + (case) [51,55] "case" + expansion [56,62] "${out}" + (${) [56,58] "${" + variable_name [58,61] "out" + (}) [61,62] "}" + (in) [63,65] "in" + case_item [66,118] "*\"not supported\"*|\\\n*\"operation not s..." + extglob_pattern [66,67] "*" + string [67,82] "\"not supported\"" + (") [67,68] "\"" + string_content [68,81] "not supported" + (") [81,82] "\"" + extglob_pattern [82,83] "*" + (|) [83,84] "|" + concatenation [86,113] "*\"operation not supported\"*" + word [86,87] "*" + string [87,112] "\"operation not supported\"" + (") [87,88] "\"" + string_content [88,111] "operation not supported" + (") [111,112] "\"" + word [112,113] "*" + ()) [113,114] ")" + (;;) [116,118] ";;" + (esac) [119,123] "esac" + case_statement [125,172] "case $1 in\n-o)\n\towner=$2\n\tshift\n\t;;\n-..." + (case) [125,129] "case" + simple_expansion [130,132] "$1" + ($) [130,131] "$" + variable_name [131,132] "1" + (in) [133,135] "in" + case_item [136,160] "-o)\n\towner=$2\n\tshift\n\t;;" + word [136,138] "-o" + ()) [138,139] ")" + variable_assignment [141,149] "owner=$2" + variable_name [141,146] "owner" + (=) [146,147] "=" + simple_expansion [147,149] "$2" + ($) [147,148] "$" + variable_name [148,149] "2" + command [151,156] "shift" + command_name [151,156] "shift" + word [151,156] "shift" + (;;) [158,160] ";;" + case_item [161,167] "-g) ;;" + word [161,163] "-g" + ()) [163,164] ")" + (;;) [165,167] ";;" + (esac) [168,172] "esac" + test_command [174,193] "[[ a == \\\"+(?)\\\" ]]" + ([[) [174,176] "[[" + binary_expression [177,190] "a == \\\"+(?)\\\"" + word [177,178] "a" + (==) [179,181] "==" + extglob_pattern [182,190] "\\\"+(?)\\\"" + (]]) [191,193] "]]" + +=== +@known-diff dollar-backtick-substitution: corpus statements.txt::Command substution with $ and backticks +$(eval echo $`echo ${foo}`) +--- +hasError: false +program [0,27] "$(eval echo $`echo ${foo}`)" + command [0,27] "$(eval echo $`echo ${foo}`)" + command_name [0,27] "$(eval echo $`echo ${foo}`)" + command_substitution [0,27] "$(eval echo $`echo ${foo}`)" + ($() [0,2] "$(" + command [2,26] "eval echo $`echo ${foo}`" + command_name [2,6] "eval" + word [2,6] "eval" + word [7,11] "echo" + concatenation [12,26] "$`echo ${foo}`" + ($) [12,13] "$" + command_substitution [13,26] "`echo ${foo}`" + (`) [13,14] "`" + command [14,25] "echo ${foo}" + command_name [14,18] "echo" + word [14,18] "echo" + expansion [19,25] "${foo}" + (${) [19,21] "${" + variable_name [21,24] "foo" + (}) [24,25] "}" + (`) [25,26] "`" + ()) [26,27] ")" +=== +=== diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/literals.txt b/packages/tree-sitter-bash/test/fixtures/corpus/literals.txt new file mode 100644 index 0000000000..61543ca3b4 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/literals.txt @@ -0,0 +1,1371 @@ +================================================================================ +Literal words +================================================================================ + +echo a +echo a b + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word) + (word))) + +================================================================================ +Words with special characters +================================================================================ + +echo {o[k]} +echo }}} +echo ]]] === + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word) + (word) + (word) + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word)) + (word))) + +================================================================================ +Simple variable expansions +================================================================================ + +echo $abc + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (simple_expansion + (variable_name)))) + +================================================================================ +Special variable expansions +================================================================================ + +echo $# $* $@ $! + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (simple_expansion + (special_variable_name)) + (simple_expansion + (special_variable_name)) + (simple_expansion + (special_variable_name)) + (simple_expansion + (special_variable_name)))) + +================================================================================ +Variable expansions +================================================================================ + +echo ${} +echo ${#} +echo ${var1#*#} +echo ${!abc} +echo ${abc} +echo ${abc:-def} +echo ${abc:+ghi} +echo ${abc:- } +echo ${abc: +} +echo ${abc,?} +echo ${abc^^b} +echo ${abc@U} +echo ${abc:- -quiet} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (word)) (expansion)) + (command (command_name (word)) (expansion)) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name) (word))) + (command (command_name (word)) (expansion (variable_name) (word))) + (command (command_name (word)) (expansion (variable_name) (word))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name) (word)))) + +================================================================================ +Variable expansions with operators +================================================================================ + +A="${B[0]# }" +C="${D/#* -E /}" +F="${G%% *}" +H="${I#*;}" +J="${K##*;}" +L="${M%;*}" +N="${O%%;*}" +P="${Q%|*}" +R="${S%()}" +T="${U%(}" +V="${W%)}" +X="${Y%<}" +Z="${A#*}" +C="${D%*}" +F="${#!}" +G=${H,,[I]} +J=${K^^[L]} +L="${M/'N'*/O}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (string + (expansion + (subscript + (variable_name) + (number)) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion))) + (variable_assignment + (variable_name) + (expansion + (variable_name) + (regex))) + (variable_assignment + (variable_name) + (expansion + (variable_name) + (regex))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex) + (word))))) + +================================================================================ +More Variable expansions with operators +================================================================================ + +${parameter-default} +${parameter:-default} +${parameter=default} +${parameter:=default} +${parameter+alt_value} +${parameter:+alt_value} +${parameter?err_msg} +${parameter:?err_msg} +${var%Pattern} +${var%%Pattern} +${var:pos} +${var:pos:len} +${MATRIX:$(($RANDOM%${#MATRIX})):1} +${PKG_CONFIG_LIBDIR:-${ESYSROOT}/usr/$(get_libdir)/pkgconfig} +${ver_str::${#ver_str}-${#not_match}} +${value#\{sd.cicd.} + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (regex)))) + (command + (command_name + (expansion + (variable_name) + (regex)))) + (command + (command_name + (expansion + (variable_name) + (variable_name)))) + (command + (command_name + (expansion + (variable_name) + (variable_name) + (variable_name)))) + (command + (command_name + (expansion + (variable_name) + (arithmetic_expansion + (binary_expression + (simple_expansion + (variable_name)) + (expansion + (variable_name)))) + (number)))) + (command + (command_name + (expansion + (variable_name) + (concatenation + (expansion (variable_name)) + (word) + (command_substitution + (command (command_name (word)))) + (word))))) + (command + (command_name + (expansion + (variable_name) + (binary_expression + (expansion (variable_name)) + (expansion (variable_name)))))) + (command + (command_name + (expansion + (variable_name) + (regex))))) + +================================================================================ +Variable expansions in strings +================================================================================ + +A="${A:-$B/c}" +A="${b=$c/$d}" +MY_PV="${PV/_pre/$'\x7e'pre}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (concatenation + (simple_expansion + (variable_name)) + (word))))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (concatenation + (simple_expansion + (variable_name)) + (word) + (simple_expansion + (variable_name)))))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex) + (concatenation (ansi_c_string) (word)))))) + +================================================================================ +Variable expansions with regexes +================================================================================ + +A=${B//:;;/$'\n'} + +# escaped space +C=${D/;\ *;|} +MOFILES=${LINGUAS// /.po }.po +MY_P="${PN/aspell/aspell"${ASPELL_VERSION}"}" +pyc=${pyc//*\/} +${pv/\.} +${new_test_cp//"${old_ver_cp}"/} +${tests_to_run//"${classes}"\/} +${allarchives// /\\|} + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (expansion + (variable_name) + (regex) + (ansi_c_string))) + (comment) + (variable_assignment + (variable_name) + (expansion (variable_name) (regex))) + (variable_assignment + (variable_name) + (concatenation + (expansion (variable_name) (regex) (word)) + (word))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex) + (concatenation + (word) + (string (expansion (variable_name))))))) + (variable_assignment + (variable_name) + (expansion (variable_name) (regex))) + (command (command_name (expansion (variable_name) (regex)))) + (command + (command_name + (expansion (variable_name) (string (expansion (variable_name)))))) + (command + (command_name + (expansion (variable_name) (string (expansion (variable_name))) (regex)))) + (command (command_name (expansion (variable_name) (regex) (word))))) + +================================================================================ +Other variable expansion operators +================================================================================ + +cat ${BAR} ${ABC=def} ${GHI:?jkl} +[ "$a" != "${a#[Bc]}" ] + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (expansion + (variable_name)) + (expansion + (variable_name) + (word)) + (expansion + (variable_name) + (word))) + (test_command + (binary_expression + (string + (simple_expansion + (variable_name))) + (string + (expansion + (variable_name) + (regex)))))) + +================================================================================ +Variable Expansions: Length +================================================================================ + +${parameter:-1} + +${parameter: -1} + +${parameter:(-1)} + +${matrix:$(($random%${#matrix})):1} + +"${_component_to_single:${len}:2}" + +"${PN::-1}" + +${trarr:$(ver_cut 2):1} + +${comp[@]:start:end*2-start} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name) (number)))) + (command + (command_name + (expansion (variable_name) (parenthesized_expression (unary_expression (number)))))) + (command + (command_name + (expansion + (variable_name) + (arithmetic_expansion + (binary_expression (simple_expansion (variable_name)) (expansion (variable_name)))) + (number)))) + (command (command_name (string (expansion (variable_name) (expansion (variable_name)) (number))))) + (command (command_name (string (expansion (variable_name) (number))))) + (command + (command_name + (expansion (variable_name) (command_substitution (command (command_name (word)) (number))) (number)))) + (command + (command_name + (expansion + (subscript (variable_name) (word)) + (variable_name) + (binary_expression (binary_expression (variable_name) (number)) (variable_name)))))) + +================================================================================ +Variable Expansions with operators +================================================================================ + +${parameter-default} +${parameter- default} +${!varprefix*} +${!varprefix@} +${parameter@U} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name))))) + +================================================================================ +Variable Expansions: Bizarre Cases +================================================================================ + +${!#} +${!# } +${!##} +${!## } +${!##/} +# here be dragons +echo "${kw}? ( ${cond:+${cond}? (} ${baseuri}-${ver}-${kw}.${suff} ${cond:+) })" + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion))) + (command (command_name (expansion))) + (command (command_name (expansion))) + (command (command_name (expansion))) + (command (command_name (expansion (special_variable_name) (regex)))) + (comment) + (command + (command_name (word)) + (string + (expansion (variable_name)) + (string_content) + (expansion + (variable_name) + (concatenation (expansion (variable_name)) (word))) + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (expansion (variable_name) (word)) + (string_content)))) + +================================================================================ +Variable Expansions: Weird Cases +================================================================================ + +${completions[*]} +${=1} +${2?} +${p_key#*=} +${abc:- } +${B[0]# } +${to_enables[0]##*/} +exec "${0#-}" --rcfile "${BASH_IT_BASHRC:-${HOME?}/.bashrc}" +recho "TDEFAULTS = ${selvecs:+-DSELECT_VECS=\"$selvecs\"}" +local msg="${2:-command '$1' does not exist}" +${cdir:+#} +${dict_langs:+;} +${UTIL_LINUX_LIBC[@]/%/? ( sys-apps/util-linux )} +${id}${2+ ${2}} +${BRANDING_GCC_PKGVERSION/(/(Gentoo ${PVR}${extvers}, } # look at that parenthesis! +some-command ${foo:+--arg <(printf '%s\n' "$foo")} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion (subscript (variable_name) (word))))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name) (regex)))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (subscript (variable_name) (number)) (regex)))) + (command (command_name (expansion (subscript (variable_name) (number)) (regex)))) + (command + (command_name (word)) + (string (expansion (special_variable_name) (regex))) + (word) + (string (expansion (variable_name) (concatenation (expansion (variable_name)) (word))))) + (command + (command_name (word)) + (string + (string_content) + (expansion (variable_name) (concatenation (word) (simple_expansion (variable_name)) (word))))) + (declaration_command + (variable_assignment + (variable_name) + (string (expansion (variable_name) (concatenation (word) (raw_string) (word)))))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (subscript (variable_name) (word)) (word)))) + (command + (command_name + (concatenation (expansion (variable_name)) (expansion (variable_name) (expansion (variable_name)))))) + (command + (command_name + (expansion + (variable_name) + (regex) + (concatenation (word) (expansion (variable_name)) (expansion (variable_name)) (word))))) + (comment) + (command + (command_name (word)) + (expansion + (variable_name) + (concatenation + (word) + (process_substitution + (command + (command_name (word)) + (raw_string) + (string (simple_expansion (variable_name))))))))) + +================================================================================ +Variable Expansions: Regex +================================================================================ + +A=${B//:;;/$'\n'} +C="${D/#* -E /}" +BASH_IT_GIT_URL="${BASH_IT_GIT_URL/git@/https://}" +10#${command_start##*.} +echo ${LIB_DEPEND//\[static-libs(+)]} +${ALL_LLVM_TARGETS[@]/%/(-)?} +filterdiff -p1 ${paths[@]/#/-i } +${cflags//-O? /$(get-flag O) } +curf="${f%'-roff2html'*}.html" +reff="${f/'-roff2html'*/'-ref'}.html" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment (variable_name) (expansion (variable_name) (regex) (ansi_c_string))) + (variable_assignment (variable_name) (string (expansion (variable_name) (regex)))) + (variable_assignment (variable_name) (string (expansion (variable_name) (regex) (word)))) + (command (command_name (number (expansion (variable_name) (regex))))) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (expansion (subscript (variable_name) (word)) (word)))) + (command (command_name (word)) (word) (expansion (subscript (variable_name) (word)) (word))) + (command + (command_name + (expansion (variable_name) (regex) (command_substitution (command (command_name (word)) (word))) (word)))) + (variable_assignment + (variable_name) + (string (expansion (variable_name) (raw_string) (regex)) (string_content))) + (variable_assignment + (variable_name) + (string (expansion (variable_name) (regex) (raw_string)) (string_content)))) + +================================================================================ +Words ending with '$' +================================================================================ + +grep ^${var}$ + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (concatenation + (word) + (expansion + (variable_name))))) + +================================================================================ +Command substitutions +================================================================================ + +echo `echo hi` +echo `echo hi; echo there` +echo $(echo $(echo hi)) +echo $(< some-file) + +# both of these are concatenations! +echo `echo otherword`word +echo word`echo otherword` + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (word)))) + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word)))) + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (word)))))) + (command + (command_name + (word)) + (command_substitution + (file_redirect + (word)))) + (comment) + (command + (command_name + (word)) + (concatenation + (command_substitution + (command + (command_name + (word)) + (word))) + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (command_substitution + (command + (command_name + (word)) + (word)))))) + +================================================================================ +Process substitutions +================================================================================ + +wc -c <(echo abc && echo def) +wc -c <(echo abc; echo def) +echo abc > >(wc -c) + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (word) + (process_substitution + (list + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word))))) + (command + (command_name + (word)) + (word) + (process_substitution + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word)))) + (redirected_statement + (command + (command_name + (word)) + (word)) + (file_redirect + (process_substitution + (command + (command_name + (word)) + (word)))))) + +================================================================================ +Single quoted strings +================================================================================ + +echo 'a b' 'c d' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (raw_string) + (raw_string))) + +================================================================================ +Double quoted strings +================================================================================ + +echo "a" "b" +echo "a ${b} c" "d $e" + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string (string_content)) + (string (string_content))) + (command + (command_name + (word)) + (string + (string_content) + (expansion + (variable_name)) + (string_content)) + (string + (string_content) + (simple_expansion + (variable_name))))) + +================================================================================ +Strings containing command substitutions +================================================================================ + +find "`dirname $file`" -name "$base"'*' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string + (command_substitution + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + (word) + (concatenation + (string + (simple_expansion + (variable_name))) + (raw_string)))) + +================================================================================ +Strings containing escape sequence +================================================================================ + +echo "\"The great escape\`\${var}" + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string (string_content)))) + +================================================================================ +Strings containing special characters +================================================================================ + +echo "s/$/'/" +echo "#" +echo "s$" + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string (string_content) (string_content))) + (command + (command_name + (word)) + (string (string_content))) + (command + (command_name + (word)) + (string (string_content)))) + +================================================================================ +Strings with ANSI-C quoting +================================================================================ + +echo $'Here\'s Johnny!\r\n' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (ansi_c_string))) + +================================================================================ +Arrays and array expansions +================================================================================ + +a=() +b=(1 2 3) + +echo ${a[@]} +echo ${#b[@]} + +a[$i]=50 +a+=(foo "bar" $(baz)) + +printf " %-9s" "${seq0:-(default)}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (array)) + (variable_assignment + (variable_name) + (array + (number) + (number) + (number))) + (command + (command_name + (word)) + (expansion + (subscript + (variable_name) + (word)))) + (command + (command_name + (word)) + (expansion + (subscript + (variable_name) + (word)))) + (variable_assignment + (subscript + (variable_name) + (simple_expansion + (variable_name))) + (number)) + (variable_assignment + (variable_name) + (array + (word) + (string (string_content)) + (command_substitution + (command + (command_name + (word)))))) + (command + (command_name + (word)) + (string (string_content)) + (string + (expansion + (variable_name) + (array + (word)))))) + +================================================================================ +Escaped characters in strings +================================================================================ + +echo -ne "\033k$1\033\\" > /dev/stderr + +-------------------------------------------------------------------------------- + +(program + (redirected_statement + (command + (command_name + (word)) + (word) + (string + (string_content) + (simple_expansion + (variable_name)) + (string_content))) + (file_redirect + (word)))) + +================================================================================ +Words containing bare '#' +================================================================================ + +curl -# localhost #comment without space +nix build nixpkgs#hello -v # comment with space + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (word) + (word)) + (comment) + (command + (command_name + (word)) + (word) + (word) + (word)) + (comment)) + +================================================================================ +Words containing # that are not comments +================================================================================ + +echo 'word'#not-comment # a legit comment +echo $(uname -a)#not-comment # a legit comment +echo `uname -a`#not-comment # a legit comment +echo $hey#not-comment # a legit comment +var=#not-comment # a legit comment +echo "'$var'" # -> '#not-comment' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (concatenation + (raw_string) + (word))) + (comment) + (command + (command_name + (word)) + (concatenation + (command_substitution + (command + (command_name + (word)) + (word))) + (word))) + (comment) + (command + (command_name + (word)) + (concatenation + (command_substitution + (command + (command_name + (word)) + (word))) + (word))) + (comment) + (command + (command_name + (word)) + (concatenation + (simple_expansion + (variable_name)) + (word))) + (comment) + (variable_assignment + (variable_name) + (word)) + (comment) + (command + (command_name + (word)) + (string + (string_content) + (simple_expansion + (variable_name)) + (string_content))) + (comment)) + +================================================================================ +Variable assignments immediately followed by a terminator +================================================================================ + +loop=; variables=& here=;; + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name)) + (variable_assignment + (variable_name)) + (variable_assignment + (variable_name))) + +================================================================================ +Multiple variable assignments +================================================================================ + +component_type="${1}" item_name="${2?}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignments + (variable_assignment + (variable_name) + (string + (expansion + (variable_name)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name)))))) + +================================================================================ +Arithmetic expansions +================================================================================ + +echo $((1 + 2 - 3 * 4 / 5)) +a=$((6 % 7 ** 8 << 9 >> 10 & 11 | 12 ^ 13)) +$(((${1:-${SECONDS}} % 12) + 144)) +((foo=0)) +echo $((bar=1)) +echo $((-1, 1)) +echo $((! -a || ~ +b || ++c || --d)) +echo $((foo-- || bar++)) +(("${MULTIBUILD_VARIANTS}" > 1)) +$(("$(stat --printf '%05a' "${save_file}")" & 07177)) +soft_errors_count=$[soft_errors_count + 1] + +-------------------------------------------------------------------------------- + +(program + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression + (binary_expression (number) (number)) + (binary_expression + (binary_expression (number) (number)) + (number))))) + (variable_assignment + (variable_name) + (arithmetic_expansion + (binary_expression + (binary_expression + (binary_expression + (binary_expression + (binary_expression + (number) + (binary_expression (number) (number))) + (number)) + (number)) + (number)) + (binary_expression (number) (number))))) + (command + (command_name + (arithmetic_expansion + (binary_expression + (parenthesized_expression + (binary_expression + (expansion + (variable_name) + (expansion (variable_name))) + (number))) + (number))))) + (command + (command_name + (arithmetic_expansion + (binary_expression (variable_name) (number))))) + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression (variable_name) (number)))) + (command + (command_name (word)) + (arithmetic_expansion (unary_expression (number)) (number))) + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression + (binary_expression + (binary_expression + (unary_expression (unary_expression (variable_name))) + (unary_expression (unary_expression (variable_name)))) + (unary_expression (variable_name))) + (unary_expression (variable_name))))) + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression + (postfix_expression + (variable_name)) + (postfix_expression + (variable_name))))) + (command + (command_name + (arithmetic_expansion + (binary_expression + (string + (expansion + (variable_name))) + (number))))) + (command + (command_name + (arithmetic_expansion + (binary_expression + (string + (command_substitution + (command + (command_name + (word)) + (word) + (raw_string) + (string + (expansion + (variable_name)))))) + (number))))) + (variable_assignment + (variable_name) + (arithmetic_expansion + (binary_expression + (variable_name) + (number))))) + +================================================================================ +Concatenation with double backticks +================================================================================ + +main() { + local foo="asd"` + `"fgh" +} + +--- + +(program + (function_definition + (word) + (compound_statement + (declaration_command + (variable_assignment + (variable_name) + (concatenation + (string (string_content)) + (string (string_content)))))))) + +================================================================================ +Brace expressions and lookalikes +================================================================================ + +echo {1..2} +echo {0..5} +echo {0..2 # not a brace expression +echo }{0..2} +echo {0..n} # not a brace expression +echo {0..n..2} # not a brace expression +echo {0..2}{1..2} + +--- + +(program + (command + (command_name (word)) + (brace_expression (number) (number))) + (command + (command_name (word)) + (brace_expression (number) (number))) + (command + (command_name (word)) + (concatenation (word) (word))) + (comment) + (command + (command_name (word)) + (concatenation + (word) + (brace_expression (number) (number)))) + (command + (command_name (word)) + (concatenation (word) (word) (word))) + (comment) + (command + (command_name (word)) + (concatenation (word) (word) (word))) + (comment) + (command + (command_name (word)) + (concatenation + (brace_expression (number) (number)) + (brace_expression (number) (number))))) diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/programs.txt b/packages/tree-sitter-bash/test/fixtures/corpus/programs.txt new file mode 100644 index 0000000000..cae772ea91 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/programs.txt @@ -0,0 +1,108 @@ +=============================== +Comments +=============================== + +#!/bin/bash +# hi + +--- + +(program + (comment) + (comment)) + +=============================== +Escaped newlines +=============================== + +abc \ + d \ + e + +f=g \ + h=i \ + j \ + --k + +--- + +(program + (command + (command_name + (word)) + (word) + (word)) + (command + (variable_assignment + (variable_name) + (word)) + (variable_assignment + (variable_name) + (word)) + (command_name + (word)) + (word))) + +============================= +escaped newline immediately after a char +============================= + +echo a \ + b + +echo a\ + b + +echo a\ + b\ + c + + +----------------------------- + +(program + (command + (command_name + (word)) + (word) + (word)) + (command + (command_name + (word)) + (word) + (word)) + (command + (command_name + (word)) + (word) + (word) + (word))) + +============================= +Escaped whitespace +============================= + +echo 1 \ 2 \ 3 + +--- + +(program + (command + (command_name + (word)) + (number) + (number) + (number))) + +==================================== +Files without trailing terminators +==================================== + +echo hi +--- + +(program + (command + (command_name + (word)) + (word))) diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/statements.txt b/packages/tree-sitter-bash/test/fixtures/corpus/statements.txt new file mode 100644 index 0000000000..b530fd4e24 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/statements.txt @@ -0,0 +1,1622 @@ +================================================================================ +Pipelines +================================================================================ + +whoami | cat +cat foo | grep -v bar +cat baz | head -n 1 + +-------------------------------------------------------------------------------- + +(program + (pipeline + (command + name: (command_name + (word))) + (command + name: (command_name + (word)))) + (pipeline + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word) + argument: (word))) + (pipeline + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word) + argument: (number)))) + + +================================================================================ +Lists +================================================================================ + +a | b && c && d; d e f || e g + +-------------------------------------------------------------------------------- + +(program + (list + (list + (pipeline + (command + (command_name + (word))) + (command + (command_name + (word)))) + (command + (command_name + (word)))) + (command + (command_name + (word)))) + (list + (command + (command_name + (word)) + (word) + (word)) + (command + (command_name + (word)) + (word)))) + +================================================================================ +While statements +================================================================================ + +while something happens; do + echo a + echo b +done + +while local name="$1" val="$2"; shift 2; do + printf "%s (%s)\n" "$val" "$name" +done + +-------------------------------------------------------------------------------- + +(program + (while_statement + condition: (command + name: (command_name + (word)) + argument: (word)) + body: (do_group + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word)))) + (while_statement + condition: (declaration_command + (variable_assignment + name: (variable_name) + value: (string + (simple_expansion + (variable_name)))) + (variable_assignment + name: (variable_name) + value: (string + (simple_expansion + (variable_name))))) + condition: (command + name: (command_name + (word)) + argument: (number)) + body: (do_group + (command + name: (command_name + (word)) + argument: (string (string_content)) + argument: (string + (simple_expansion + (variable_name))) + argument: (string + (simple_expansion + (variable_name))))))) + +================================================================================ +Until statements +================================================================================ + +until something happens; do + echo a + echo b +done + +-------------------------------------------------------------------------------- + +(program + (while_statement + condition: (command + name: (command_name + (word)) + argument: (word)) + body: (do_group + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word))))) + +================================================================================ +While statements with IO redirects +================================================================================ + +while read line; do + echo $line +done < <(cat file) + +-------------------------------------------------------------------------------- + +(program + (redirected_statement + body: (while_statement + condition: (command + name: (command_name + (word)) + argument: (word)) + body: (do_group + (command + name: (command_name + (word)) + argument: (simple_expansion + (variable_name))))) + redirect: (file_redirect + destination: (process_substitution + (command + name: (command_name + (word)) + argument: (word)))))) + +================================================================================ +For statements +================================================================================ + +for a in 1 2 $(seq 5 10); do + echo $a +done + +for ARG; do + echo $ARG + ARG='' +done + +for c in ${=1}; do + echo c +done + +-------------------------------------------------------------------------------- + +(program + (for_statement + variable: (variable_name) + value: (number) + value: (number) + value: (command_substitution + (command + name: (command_name + (word)) + argument: (number) + argument: (number))) + body: (do_group + (command + name: (command_name + (word)) + argument: (simple_expansion + (variable_name))))) + (for_statement + variable: (variable_name) + body: (do_group + (command + name: (command_name + (word)) + argument: (simple_expansion + (variable_name))) + (variable_assignment + name: (variable_name) + value: (raw_string)))) + (for_statement + variable: (variable_name) + value: (expansion + (variable_name)) + body: (do_group + (command + name: (command_name + (word)) + argument: (word))))) + +================================================================================ +Select statements +================================================================================ + +select choice in X Y $(ls); do + echo $choice + break +done + +select ARG; do + echo $ARG + ARG='' +done + +-------------------------------------------------------------------------------- + +(program + (for_statement + (variable_name) + (word) + (word) + (command_substitution + (command + (command_name + (word)))) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word))))) + (for_statement + (variable_name) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (variable_assignment + (variable_name) + (raw_string))))) + +================================================================================ +C-style for statements +================================================================================ + +for (( c=1; c<=5; c++ )) +do + echo $c +done + +for (( c=1; c<=5; c++ )) { + echo $c +} + +for (( ; ; )) +do + echo 'forever' +done + +for ((cx = 0; c = $cx / $pf, c < $wc - $k; )); do + echo "$cx" +done + +for (( i = 4;;i--)) ; do echo $i; if (( $i == 0 )); then break; fi; done + +# added post-bash-4.2 +for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done + +echo + +( for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done ) + +-------------------------------------------------------------------------------- + +(program + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (binary_expression + (word) + (number)) + (postfix_expression + (word)) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (binary_expression + (word) + (number)) + (postfix_expression + (word)) + (compound_statement + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + (c_style_for_statement + (do_group + (command + (command_name + (word)) + (raw_string)))) + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (variable_assignment + (variable_name) + (binary_expression + (simple_expansion + (variable_name)) + (simple_expansion + (variable_name)))) + (binary_expression + (word) + (binary_expression + (simple_expansion + (variable_name)) + (simple_expansion + (variable_name)))) + (do_group + (command + (command_name + (word)) + (string + (simple_expansion + (variable_name)))))) + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (postfix_expression + (word)) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (if_statement + (command + (command_name + (arithmetic_expansion + (binary_expression + (simple_expansion + (variable_name)) + (number))))) + (command + (command_name + (word)))))) + (comment) + (c_style_for_statement + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (number)))) + (binary_expression + (binary_expression + (word) + (number)) + (parenthesized_expression + (binary_expression + (word) + (number)) + (command_substitution + (redirected_statement + (list + (command + (command_name + (arithmetic_expansion + (binary_expression + (variable_name) + (number))))) + (command + (command_name + (word)) + (string))) + (file_redirect + (number))) + (command + (command_name + (word)) + (number))) + (binary_expression + (postfix_expression + (word)) + (number)))) + (binary_expression + (word) + (word)) + (do_group + (command + (command_name + (word)) + (string + (simple_expansion + (variable_name)))))) + (command + (command_name + (word))) + (subshell + (c_style_for_statement + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (number)))) + (binary_expression + (binary_expression + (word) + (number)) + (parenthesized_expression + (binary_expression + (word) + (number)) + (command_substitution + (redirected_statement + (list + (command + (command_name + (arithmetic_expansion + (binary_expression + (variable_name) + (number))))) + (command + (command_name + (word)) + (string))) + (file_redirect + (number))) + (command + (command_name + (word)) + (number))) + (binary_expression + (postfix_expression + (word)) + (number)))) + (binary_expression + (word) + (word)) + (do_group + (command + (command_name + (word)) + (string + (simple_expansion + (variable_name)))))))) + +================================================================================ +If statements +================================================================================ + +if cat some_file | grep -v ok; then + echo one +elif cat other_file | grep -v ok; then + echo two +else + exit +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + (pipeline + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word) + (word))) + (command + (command_name + (word)) + (word)) + (elif_clause + (pipeline + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word) + (word))) + (command + (command_name + (word)) + (word))) + (else_clause + (command + (command_name + (word)))))) + +================================================================================ +If statements with conditional expressions +================================================================================ + +if [ "$(uname)" == 'Darwin' ]; then + echo one +fi + +if [ a = -d ]; then + echo two +fi + +[[ abc == +(a|b|c) ]] && echo 1 +[[ abc != +(a|b|c) ]] && echo 2 + +-------------------------------------------------------------------------------- + +(program + (if_statement + (test_command + (binary_expression + (string + (command_substitution + (command + (command_name + (word))))) + (raw_string))) + (command + (command_name + (word)) + (word))) + (if_statement + (test_command + (binary_expression + (word) + (word))) + (command + (command_name + (word)) + (word))) + (list + (test_command + (binary_expression + (word) + (extglob_pattern))) + (command + (command_name + (word)) + (number))) + (list + (test_command + (binary_expression + (word) + (extglob_pattern))) + (command + (command_name + (word)) + (number)))) + +================================================================================ +If statements with negated command +================================================================================ + +if ! command -v echo; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (negated_command + (command + name: (command_name + (word)) + argument: (word) + argument: (word))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with command +================================================================================ + +if command -v echo; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (command + name: (command_name + (word)) + argument: (word) + argument: (word)) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with variable assignment by command substitution +================================================================================ + +if result=$(echo 'hello'); then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (variable_assignment + name: (variable_name) + value: (command_substitution + (command + name: (command_name + (word)) + argument: (raw_string)))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with negated variable assignment by command substitution +================================================================================ + +if ! result=$(echo 'hello'); then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (negated_command + (variable_assignment + name: (variable_name) + value: (command_substitution + (command + name: (command_name + (word)) + argument: (raw_string))))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with variable assignment +================================================================================ + +if foo=1; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (variable_assignment + name: (variable_name) + value: (number)) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with negated variable assignment +================================================================================ + +if ! foo=1; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (negated_command + (variable_assignment + name: (variable_name) + value: (number))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +Case statements +================================================================================ + +case "opt" in + a) + echo a + ;; + + b) + echo b + ;& + + c) + echo c;; +esac + +case "opt" in + (a) + echo a + ;; + + (b) + echo b + ;& + + (c) + echo c;; +esac + +case "$Z" in + ab*|cd*) ef +esac + +case $dest in + *.[1357]) + exit $? + ;; +esac + +case x in x) echo meow ;; esac + +case foo in + bar\ baz) : ;; +esac + +case "$arg" in + *([0-9])([0-9])) echo "$arg" +esac + +case ${lang} in +CMakeLists.txt | \ + cmake_modules | \ + ${PN}.pot) ;; +*) rm -r ${lang} || die ;; +esac + +-------------------------------------------------------------------------------- + +(program + (case_statement + (string (string_content)) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word)))) + (case_statement + (string (string_content)) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word)))) + (case_statement + (string (simple_expansion (variable_name))) + (case_item (extglob_pattern) (extglob_pattern) (command (command_name (word))))) + (case_statement + (simple_expansion (variable_name)) + (case_item + (concatenation (word) (word) (number) (word)) + (command (command_name (word)) (simple_expansion (special_variable_name))))) + (case_statement (word) (case_item (word) (command (command_name (word)) (word)))) + (case_statement (word) (case_item (word) (command (command_name (word))))) + (case_statement + (string (simple_expansion (variable_name))) + (case_item + (extglob_pattern) + (command (command_name (word)) (string (simple_expansion (variable_name)))))) + (case_statement + (expansion (variable_name)) + (case_item (word) (word) (concatenation (expansion (variable_name)) (word))) + (case_item + (extglob_pattern) + (list + (command (command_name (word)) (word) (expansion (variable_name))) + (command (command_name (word))))))) + +================================================================================ +Test commands +================================================================================ + +if [[ "$lsb_dist" != 'Ubuntu' || $(ver_to_int "$lsb_release") < $(ver_to_int '14.04') ]]; then + return 1 +fi + +[[ ${PV} != $(sed -n -e 's/^Version: //p' "${ED}/usr/$(get_libdir)/pkgconfig/tss2-tcti-tabrmd.pc" || die) ]] + +[[ ${f} != */@(default).vim ]] + +[[ "${MY_LOCALES}" != *en_US* || a != 2 ]] + +[[ $(LC_ALL=C $(tc-getCC) ${LDFLAGS} -Wl,--version 2>/dev/null) != @(LLD|GNU\ ld)* ]] + +[[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \ + -x "${EROOT}/usr/bin/peardev" ]] + +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] + +[[ ${SRC_URI} == */${a}* ]] + +[[ a == *_@(LIB|SYMLINK) ]] + +[[ ${1} =~ \.(lisp|lsp|cl)$ ]] + +[[ a == - ]] + +-------------------------------------------------------------------------------- + +(program + (if_statement + (test_command + (binary_expression + (binary_expression (string (simple_expansion (variable_name))) (raw_string)) + (binary_expression + (command_substitution + (command (command_name (word)) (string (simple_expansion (variable_name))))) + (command_substitution (command (command_name (word)) (raw_string)))))) + (command (command_name (word)) (number))) + (test_command + (binary_expression + (expansion (variable_name)) + (command_substitution + (list + (command + (command_name (word)) + (word) + (word) + (raw_string) + (string + (expansion (variable_name)) + (string_content) + (command_substitution (command (command_name (word)))) + (string_content))) + (command (command_name (word))))))) + (test_command (binary_expression (expansion (variable_name)) (extglob_pattern))) + (test_command + (binary_expression + (binary_expression (string (expansion (variable_name))) (extglob_pattern)) + (binary_expression (word) (number)))) + (test_command + (binary_expression + (command_substitution + (redirected_statement + (command + (variable_assignment (variable_name) (word)) + (command_name (command_substitution (command (command_name (word))))) + (expansion (variable_name)) + (word)) + (file_redirect (file_descriptor) (word)))) + (extglob_pattern))) + (test_command + (binary_expression + (unary_expression + (test_operator) + (string + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (string_content))) + (unary_expression + (test_operator) + (string (expansion (variable_name)) (string_content))))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (command_substitution + (variable_assignment (variable_name) (raw_string)) + (command + (command_name (word)) + (string (expansion (subscript (variable_name) (word)))))) + (extglob_pattern))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command (binary_expression (word) (extglob_pattern))) + (test_command (binary_expression (expansion (variable_name)) (regex))) + (test_command (binary_expression (word) (extglob_pattern)))) + +================================================================================ +Test commands with ternary +================================================================================ + +if (( 1 < 2 ? 1 : 2 )); then + return 1 +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (number) + (number)) + (number) + (number))))) + (command + (command_name + (word)) + (number)))) + +================================================================================ +Ternary expressions +================================================================================ + +$((n < 10 ? n : 10)) +$(($n < 10 ? $n : 10)) +$((${n} < 10 ? ${n} : 10)) + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (variable_name) + (number)) + (variable_name) + (number))))) + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (simple_expansion + (variable_name)) + (number)) + (simple_expansion + (variable_name)) + (number))))) + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (expansion + (variable_name)) + (number)) + (expansion + (variable_name)) + (number)))))) + +================================================================================ +Test commands with regexes +================================================================================ + +[[ "35d8b" =~ ^[0-9a-fA-F] ]] +[[ $CMD =~ (^|;)update_terminal_cwd($|;) ]] +[[ ! " ${completions[*]} " =~ " $alias_cmd " ]] +! [[ "$a" =~ ^a|b\ *c|d$ ]] +[[ "$1" =~ ^${var}${var}*=..* ]] +[[ "$1" =~ ^\-${var}+ ]] +[[ ${var1} == *${var2}* ]] +[[ "$server" =~ [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3} ]] +[[ "$primary_wins" =~ ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}) ]] +[[ -f ${x} && $(od -t x1 -N 4 "${x}") == *"7f 45 4c 46"* ]] +[[ ${PV} =~ 99999999$ ]] +[[ $1 == -- ]] +[[ " ${REPLACING_VERSIONS} " != *\ ${PVR}\ * ]] +[[ ${file} == @(*${GENTOO_PATCH_NAME}.tar.xz|*.asc|*.sig) ]] +[[ $RUBY_TARGETS != *$( eselect ruby show | awk 'NR==2' | tr -d ' ' )* ]] +[[ " ${m[0]##*/}" =~ ^(\ ${skip_files[*]/%/.*|\\} ) ]] +[[ ' boop xyz' =~ ' boop '(.*)$ ]] +[[ $b = foo* ]] + +-------------------------------------------------------------------------------- + +(program + (test_command (binary_expression (string (string_content)) (regex))) + (test_command (binary_expression (simple_expansion (variable_name)) (regex))) + (test_command + (binary_expression + (unary_expression + (string (expansion (subscript (variable_name) (word))))) + (string (simple_expansion (variable_name))))) + (negated_command (test_command (binary_expression (string (simple_expansion (variable_name))) (regex)))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command + (binary_expression + (unary_expression (test_operator) (expansion (variable_name))) + (binary_expression + (command_substitution + (command + (command_name (word)) + (word) + (word) + (word) + (number) + (string (expansion (variable_name))))) + (extglob_pattern) + (string (string_content)) + (extglob_pattern)))) + (test_command (binary_expression (expansion (variable_name)) (regex))) + (test_command (binary_expression (simple_expansion (variable_name)) (extglob_pattern))) + (test_command + (binary_expression + (string (expansion (variable_name))) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command + (binary_expression + (simple_expansion (variable_name)) + (extglob_pattern) + (command_substitution + (pipeline + (command (command_name (word)) (word) (word)) + (command (command_name (word)) (raw_string)) + (command (command_name (word)) (word) (raw_string)))) + (extglob_pattern))) + (test_command + (binary_expression (string (expansion (subscript (variable_name) (number)) (regex))) (regex))) + (test_command (binary_expression (raw_string) (regex))) + (test_command (binary_expression (simple_expansion (variable_name)) (regex)))) + +================================================================================ +Test command paren statefulness with a case glob +================================================================================ + +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] + +case ${out} in +*"not supported"*|\ +*"operation not supported"*) + ;; +esac + +case $1 in +-o) + owner=$2 + shift + ;; +-g) ;; +esac + +[[ a == \"+(?)\" ]] + +--- + +(program + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (command_substitution + (variable_assignment (variable_name) (raw_string)) + (command + (command_name (word)) + (string (expansion (subscript (variable_name) (word)))))) + (extglob_pattern))) + (case_statement + (expansion (variable_name)) + (case_item + (extglob_pattern) + (string (string_content)) + (extglob_pattern) + (concatenation (word) (string (string_content)) (word)))) + (case_statement + (simple_expansion (variable_name)) + (case_item + (extglob_pattern) + (variable_assignment (variable_name) (simple_expansion (variable_name))) + (command (command_name (word)))) + (case_item (word))) + (test_command (binary_expression (word) (extglob_pattern)))) + +================================================================================ +Subshells +================================================================================ + +( + ./start-server --port=80 +) & + +time ( cd tests && sh run-tests.sh ) + +-------------------------------------------------------------------------------- + +(program + (subshell (command (command_name (word)) (word))) + (command + (command_name (word)) + (subshell + (list + (command (command_name (word)) (word)) + (command (command_name (word)) (word)))))) + +================================================================================ +Function definitions +================================================================================ + +do_something() { + echo ok +} + +run_subshell_command() ( + true +) + +run_test_command() [[ -e foo ]] + +function do_something_else() { + a | xargs -I{} find xml/{} -type f +} + +function do_yet_another_thing { + echo ok +} 2>&1 + +do_nothing() { return 0; } + +foo::bar() { + echo what +} + +foo::baz() { + echo how +} + +assert() + if ! $1; then + return 1 + fi + +-------------------------------------------------------------------------------- + +(program + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word)))) + (function_definition + (word) + (subshell + (command + (command_name + (word))))) + (function_definition + (word) + (test_command + (unary_expression + (test_operator) + (word)))) + (function_definition + (word) + (compound_statement + (pipeline + (command + (command_name + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word)) + (word) + (concatenation + (word) + (word) + (word)) + (word) + (word))))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word))) + (file_redirect + (file_descriptor) + (number))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (number)))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word)))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word)))) + (function_definition + (word) + (if_statement + (negated_command + (command + (command_name + (simple_expansion + (variable_name))))) + (command + (command_name + (word)) + (number))))) + +================================================================================ +Variable declaration: declare & typeset +================================================================================ + +declare var1 +typeset -i -r var2=42 var3=10 + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_name)) + (declaration_command + (word) + (word) + (variable_assignment + (variable_name) + (number)) + (variable_assignment + (variable_name) + (number)))) + +================================================================================ +Variable declaration: readonly +================================================================================ + +readonly var1 +readonly var2=42 + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_name)) + (declaration_command + (variable_assignment + (variable_name) + (number)))) + +================================================================================ +Variable declaration: local +================================================================================ + +local a=42 b +local -r c +local var=word1\ word2 + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_assignment + (variable_name) + (number)) + (variable_name)) + (declaration_command + (word) + (variable_name)) + (declaration_command + (variable_assignment + (variable_name) + (word)))) + +================================================================================ +Variable declaration: export +================================================================================ + +export PATH +export FOOBAR PATH="$PATH:/usr/foobar/bin" +export $FOO:$BAR + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_name)) + (declaration_command + (variable_name) + (variable_assignment + (variable_name) + (string + (simple_expansion + (variable_name)) + (string_content)))) + (declaration_command + (concatenation + (simple_expansion + (variable_name)) + (word) + (simple_expansion + (variable_name))))) + +================================================================================ +Variable declaration: command substitution with semi-colon +================================================================================ + +_path=$( + while statement; do + cd ".." + done; + echo $PWD +) + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (command_substitution + (while_statement + (command + (command_name + (word))) + (do_group + (command + (command_name + (word)) + (string (string_content))))) + (command + (command_name + (word)) + (simple_expansion + (variable_name)))))) + +=========================================================== +Command substution with $ and backticks +=========================================================== + +$(eval echo $`echo ${foo}`) + +--- + +(program + (command + (command_name + (command_substitution + (command + (command_name + (word)) + (word) + (command_substitution + (command + (command_name + (word)) + (expansion + (variable_name))))))))) + +================================================================================ +Expressions passed to declaration commands +================================================================================ + +export "$(echo ${key} | tr [:lower:] [:upper:])=${p_key#*=}" + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (string + (command_substitution + (pipeline + (command + (command_name + (word)) + (expansion + (variable_name))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word)) + (concatenation + (word) + (word) + (word))))) + (string_content) + (expansion + (variable_name) + (regex))))) + +================================================================================ +Unset commands +================================================================================ + +unset A +unset "$variable_name" +unsetenv -f ONE TWO + +-------------------------------------------------------------------------------- + +(program + (unset_command + (variable_name)) + (unset_command + (string + (simple_expansion + (variable_name)))) + (unset_command + (word) + (variable_name) + (variable_name))) + +================================================================================ +Compound statements +================================================================================ + +a () { + ls || { echo "b"; return 0; } + echo c +} + +{ echo "a" + echo "b" +} >&2 + +-------------------------------------------------------------------------------- + +(program + (function_definition + (word) + (compound_statement + (list + (command + (command_name + (word))) + (compound_statement + (command + (command_name + (word)) + (string (string_content))) + (command + (command_name + (word)) + (number)))) + (command + (command_name + (word)) + (word)))) + (redirected_statement + (compound_statement + (command + (command_name + (word)) + (string (string_content))) + (command + (command_name + (word)) + (string (string_content)))) + (file_redirect + (number)))) + +================================================================================ +If condition with subshell +================================================================================ + +if (echo $BASHPID; true); then echo $BASHPID; fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + (subshell + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word)))) + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + +================================================================================ +While condition with subshell +================================================================================ + +while (echo $BASHPID; true); do echo $BASHPID; break; done + +-------------------------------------------------------------------------------- + +(program + (while_statement + (subshell + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word)))) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word)))))) diff --git a/packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt b/packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt new file mode 100644 index 0000000000..f2583d371a --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt @@ -0,0 +1,236 @@ +@match: for ((i=0;i<3;i++)); do echo $i; done +for ((i=0;i<3;i++)); do echo $i; done +=== +@match: for (( i = 0; i < 10; i += 2 )); do x; done +for (( i = 0; i < 10; i += 2 )); do x; done +=== +@match: echo $((1 + 2 * 3)) +echo $((1 + 2 * 3)) +=== +@match: echo $((a > b ? a : b)) +echo $((a > b ? a : b)) +=== +@match: echo $(( (1+2) ** 3 )) +echo $(( (1+2) ** 3 )) +=== +@match: echo $((i++ + --j)) +echo $((i++ + --j)) +=== +@match: echo $((-x + ~y)) +echo $((-x + ~y)) +=== +@match: echo $((arr[0] + 1)) +echo $((arr[0] + 1)) +=== +@match: echo $((x += 5)) +echo $((x += 5)) +=== +@match: echo $((1, 2, 3)) +echo $((1, 2, 3)) +=== +@match: ((x = y + 1)) +((x = y + 1)) +=== +@match: echo $((n % 2 == 0)) +echo $((n % 2 == 0)) +=== +@match: echo $((2 ** 3 ** 2)) +echo $((2 ** 3 ** 2)) +=== +@match: echo $((x = y = 1)) +echo $((x = y = 1)) +=== +@match: echo $(( +echo $(( +1 + +2 +)) +=== +@match: (( ls )) +(( ls )) +=== +@match: ((x=1)); echo $x +((x=1)); echo $x +=== +@match: for ((;;)); do x; done +for ((;;)); do x; done +=== +@match: for ((i=0,j=1; i<3; i++,j--)); do x; done +for ((i=0,j=1; i<3; i++,j--)); do x; done +=== +@match: for ((i=0;i<3;i++)) { echo $i; } +for ((i=0;i<3;i++)) { echo $i; } +=== +@match: echo $[1+2] +echo $[1+2] +=== +@match: echo $((x[1]+y[z])) +echo $((x[1]+y[z])) +=== +@match: ((a++)) +((a++)) +=== +@match: ((a--)) +((a--)) +=== +@match: ((a++ + b)) +((a++ + b)) +=== +@match: ((b + a++)) +((b + a++)) +=== +@known-diff arithmetic-hex: echo $((0x1F)) +echo $((0x1F)) +--- +hasError: false +program [0,14] "echo $((0x1F))" + command [0,14] "echo $((0x1F))" + command_name [0,4] "echo" + word [0,4] "echo" + arithmetic_expansion [5,14] "$((0x1F))" + ($(() [5,8] "$((" + number [8,12] "0x1F" + ())) [12,14] "))" +=== +@match: echo ((a++)) +echo ((a++)) +=== +@match: ((a[i]++)) +((a[i]++)) +=== +@match: ! ((a++)) +! ((a++)) +=== +@match: ((ab-cd++)) +((ab-cd++)) +=== +@match: echo $((1+2)) more +echo $((1+2)) more +=== +@match: $(((${1:-${SECONDS}} % 12) + 144)) +$(((${1:-${SECONDS}} % 12) + 144)) +=== +@match: echo $((-1, 1)) +echo $((-1, 1)) +=== +@match: echo $((! -a || ~ +b || ++c || --d)) +echo $((! -a || ~ +b || ++c || --d)) +=== +@match: (("${MULTIBUILD_VARIANTS}" > 1)) +(("${MULTIBUILD_VARIANTS}" > 1)) +=== +@match: $(("$(stat --printf '%05a' "${save_file}")" & 07177)) +$(("$(stat --printf '%05a' "${save_file}")" & 07177)) +=== +@match: soft_errors_count=$[soft_errors_count + 1] +soft_errors_count=$[soft_errors_count + 1] +=== +@known-diff cfor-compound-assign-negative: for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || pr... +for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done +--- +hasError: false +program [0,121] "for (( i = j = k = 1; i % 9 || (j *= ..." + c_style_for_statement [0,121] "for (( i = j = k = 1; i % 9 || (j *= ..." + (for) [0,3] "for" + ((() [4,6] "((" + variable_assignment [7,20] "i = j = k = 1" + variable_name [7,8] "i" + (=) [9,10] "=" + variable_assignment [11,20] "j = k = 1" + variable_name [11,12] "j" + (=) [13,14] "=" + variable_assignment [15,20] "k = 1" + variable_name [15,16] "k" + (=) [17,18] "=" + number [19,20] "1" + (;) [20,21] ";" + binary_expression [22,90] "i % 9 || (j *= -1, $( ((i%9)) || prin..." + binary_expression [22,27] "i % 9" + word [22,23] "i" + (%) [24,25] "%" + number [26,27] "9" + (||) [28,30] "||" + parenthesized_expression [31,90] "(j *= -1, $( ((i%9)) || printf \" \" >&..." + (() [31,32] "(" + binary_expression [32,39] "j *= -1" + word [32,33] "j" + (*=) [34,36] "*=" + unary_expression [37,39] "-1" + (-) [37,38] "-" + number [38,39] "1" + (,) [39,40] "," + command_substitution [41,78] "$( ((i%9)) || printf \" \" >&2; echo 0)" + ($() [41,43] "$(" + list [44,69] "((i%9)) || printf \" \" >&2" + command [44,51] "((i%9))" + command_name [44,51] "((i%9))" + arithmetic_expansion [44,51] "((i%9))" + ((() [44,46] "((" + binary_expression [46,49] "i%9" + variable_name [46,47] "i" + (%) [47,48] "%" + number [48,49] "9" + ())) [49,51] "))" + (||) [52,54] "||" + redirected_statement [55,69] "printf \" \" >&2" + command [55,65] "printf \" \"" + command_name [55,61] "printf" + word [55,61] "printf" + string [62,65] "\" \"" + (") [62,63] "\"" + string_content [63,64] " " + (") [64,65] "\"" + file_redirect [66,69] ">&2" + (>&) [66,68] ">&" + number [68,69] "2" + (;) [69,70] ";" + command [71,77] "echo 0" + command_name [71,75] "echo" + word [71,75] "echo" + number [76,77] "0" + ()) [77,78] ")" + (,) [78,79] "," + binary_expression [80,89] "k++ <= 10" + postfix_expression [80,83] "k++" + word [80,81] "k" + (++) [81,83] "++" + (<=) [84,86] "<=" + number [87,89] "10" + ()) [89,90] ")" + (;) [90,91] ";" + binary_expression [92,98] "i += j" + word [92,93] "i" + (+=) [94,96] "+=" + word [97,98] "j" + ())) [99,101] "))" + do_group [102,121] "do\nprintf \"$i\"\ndone" + (do) [102,104] "do" + command [105,116] "printf \"$i\"" + command_name [105,111] "printf" + word [105,111] "printf" + string [112,116] "\"$i\"" + (") [112,113] "\"" + simple_expansion [113,115] "$i" + ($) [113,114] "$" + variable_name [114,115] "i" + (") [115,116] "\"" + (done) [117,121] "done" +=== +@match: ((x = -1)) +((x = -1)) +=== +@match: ((x += -1)) +((x += -1)) +=== +@match: echo $((-1)) +echo $((-1)) +=== +@match: echo $((x * -1)) +echo $((x * -1)) +=== +@match: echo $((x=-1)) +echo $((x=-1)) +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/case.txt b/packages/tree-sitter-bash/test/fixtures/differential/case.txt new file mode 100644 index 0000000000..bbb781340e --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/case.txt @@ -0,0 +1,436 @@ +@match: case $x in +case $x in + a) echo A ;; + b|c) echo BC ;& + *) echo other ;; +esac +=== +@match: case $x in (a) x ;; esac +case $x in (a) x ;; esac +=== +@match: case $x in a|b) x ;; esac +case $x in a|b) x ;; esac +=== +@match: case $x in foo*|bar) x ;; esac +case $x in foo*|bar) x ;; esac +=== +@match: case $x in [a-z]) x ;; esac +case $x in [a-z]) x ;; esac +=== +@match: case $x in "a") x ;; esac +case $x in "a") x ;; esac +=== +@match: case $x in $v|${w}) x ;; esac +case $x in $v|${w}) x ;; esac +=== +@known-diff recovery-partial-nodes: case $x in a) x ;;& esac +case $x in a) x ;;& esac +--- +hasError: true +program [0,24] "case $x in a) x ;;& esac" + case_statement [0,24] "case $x in a) x ;;& esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,19] "a) x ;;&" + word [11,12] "a" + ()) [12,13] ")" + command [14,15] "x" + command_name [14,15] "x" + word [14,15] "x" + (;;&) [16,19] ";;&" + (esac) [20,24] "esac" +=== +@match: case $x in a) x esac +case $x in a) x esac +=== +@match: case $x in esac +case $x in esac +=== +@match: case x in a) ;; esac +case x in a) ;; esac +=== +@match: case $x in +case $x in +a) y ;; +esac +=== +@match: case $x in a) echo esac ;; esac +case $x in a) echo esac ;; esac +=== +@match: case $x in a) x +case $x in a) x +esac +=== +@match: x=$(case y in a) 1;; esac) +x=$(case y in a) 1;; esac) +=== +@match: x=$(case y in (b) 2;; esac) +x=$(case y in (b) 2;; esac) +=== +@match: diff <(case y in a) 1;; esac) other +diff <(case y in a) 1;; esac) other +=== +@match: case $x in [1]) x ;; esac +case $x in [1]) x ;; esac +=== +@known-diff recovery-partial-nodes: if a; then case x in b) 1;; esac elif c; then d; fi +if a; then case x in b) 1;; esac elif c; then d; fi +--- +hasError: true +program [0,51] "if a; then case x in b) 1;; esac elif..." + if_statement [0,51] "if a; then case x in b) 1;; esac elif..." + (if) [0,2] "if" + command [3,4] "a" + command_name [3,4] "a" + word [3,4] "a" + (;) [4,5] ";" + (then) [6,10] "then" + case_statement [11,32] "case x in b) 1;; esac" + (case) [11,15] "case" + word [16,17] "x" + (in) [18,20] "in" + case_item [21,27] "b) 1;;" + word [21,22] "b" + ()) [22,23] ")" + command [24,25] "1" + command_name [24,25] "1" + number [24,25] "1" + (;;) [25,27] ";;" + (esac) [28,32] "esac" + elif_clause [33,48] "elif c; then d;" + (elif) [33,37] "elif" + command [38,39] "c" + command_name [38,39] "c" + word [38,39] "c" + (;) [39,40] ";" + (then) [41,45] "then" + command [46,47] "d" + command_name [46,47] "d" + word [46,47] "d" + (;) [47,48] ";" + (fi) [49,51] "fi" +=== +@match: case $dest in +case $dest in + *.[1357]) + exit $? + ;; +esac +=== +@match: case "$arg" in +case "$arg" in + *([0-9])([0-9])) echo "$arg" +esac +=== +@known-diff case-pattern-after-continuation: case ${lang} in +case ${lang} in +CMakeLists.txt | \ + cmake_modules | \ + ${PN}.pot) ;; +*) rm -r ${lang} || die ;; +esac +--- +hasError: false +program [0,100] "case ${lang} in\nCMakeLists.txt | \\\n\tc..." + case_statement [0,100] "case ${lang} in\nCMakeLists.txt | \\\n\tc..." + (case) [0,4] "case" + expansion [5,12] "${lang}" + (${) [5,7] "${" + variable_name [7,11] "lang" + (}) [11,12] "}" + (in) [13,15] "in" + case_item [16,68] "CMakeLists.txt | \\\n\tcmake_modules | \\..." + word [16,30] "CMakeLists.txt" + (|) [31,32] "|" + extglob_pattern [36,49] "cmake_modules" + (|) [50,51] "|" + concatenation [55,64] "${PN}.pot" + expansion [55,60] "${PN}" + (${) [55,57] "${" + variable_name [57,59] "PN" + (}) [59,60] "}" + word [60,64] ".pot" + ()) [64,65] ")" + (;;) [66,68] ";;" + case_item [69,95] "*) rm -r ${lang} || die ;;" + extglob_pattern [69,70] "*" + ()) [70,71] ")" + list [72,92] "rm -r ${lang} || die" + command [72,85] "rm -r ${lang}" + command_name [72,74] "rm" + word [72,74] "rm" + word [75,77] "-r" + expansion [78,85] "${lang}" + (${) [78,80] "${" + variable_name [80,84] "lang" + (}) [84,85] "}" + (||) [86,88] "||" + command [89,92] "die" + command_name [89,92] "die" + word [89,92] "die" + (;;) [93,95] ";;" + (esac) [96,100] "esac" +=== +@match: case ${out} in +case ${out} in +*"not supported"*|\ +*"operation not supported"*) + ;; +esac +=== +@known-diff case-short-option-pattern: case $1 in +case $1 in +-o) + owner=$2 + shift + ;; +-g) ;; +esac +--- +hasError: false +program [0,47] "case $1 in\n-o)\n\towner=$2\n\tshift\n\t;;\n-..." + case_statement [0,47] "case $1 in\n-o)\n\towner=$2\n\tshift\n\t;;\n-..." + (case) [0,4] "case" + simple_expansion [5,7] "$1" + ($) [5,6] "$" + variable_name [6,7] "1" + (in) [8,10] "in" + case_item [11,35] "-o)\n\towner=$2\n\tshift\n\t;;" + word [11,13] "-o" + ()) [13,14] ")" + variable_assignment [16,24] "owner=$2" + variable_name [16,21] "owner" + (=) [21,22] "=" + simple_expansion [22,24] "$2" + ($) [22,23] "$" + variable_name [23,24] "2" + command [26,31] "shift" + command_name [26,31] "shift" + word [26,31] "shift" + (;;) [33,35] ";;" + case_item [36,42] "-g) ;;" + word [36,38] "-g" + ()) [38,39] ")" + (;;) [40,42] ";;" + (esac) [43,47] "esac" +=== +@match: case x in *.txt) ;; esac +case x in *.txt) ;; esac +=== +@match: case x in [abc]) ;; esac +case x in [abc]) ;; esac +=== +@match: case x in a|b) ;; esac +case x in a|b) ;; esac +=== +@match: case x in -o|--output) ;; esac +case x in -o|--output) ;; esac +=== +@match: case x in x) ;; esac +case x in x) ;; esac +=== +@match: case x in foo.txt) ;; esac +case x in foo.txt) ;; esac +=== +@match: case x in *foo) ;; esac +case x in *foo) ;; esac +=== +@match: case x in foo*) ;; esac +case x in foo*) ;; esac +=== +@match: case x in *) ;; esac +case x in *) ;; esac +=== +@match: case x in ?) ;; esac +case x in ?) ;; esac +=== +@match: case x in *.*) ;; esac +case x in *.*) ;; esac +=== +@match: case x in foo|bar) ;; esac +case x in foo|bar) ;; esac +=== +@match: case x in foo.txt|*.log) ;; esac +case x in foo.txt|*.log) ;; esac +=== +@match: case x in [a-z]*) ;; esac +case x in [a-z]*) ;; esac +=== +@match: case x in *.[1357]) ;; esac +case x in *.[1357]) ;; esac +=== +@match: case x in -o) ;; esac +case x in -o) ;; esac +=== +@match: case x in --output) ;; esac +case x in --output) ;; esac +=== +@match: case x in +(a|b)) ;; esac +case x in +(a|b)) ;; esac +=== +@match: case x in *(a|b)) ;; esac +case x in *(a|b)) ;; esac +=== +@match: case x in @(a|b)) ;; esac +case x in @(a|b)) ;; esac +=== +@known-diff case-rejected-group: case x in x@(y)) ;; esac +case x in x@(y)) ;; esac +--- +hasError: true +program [0,24] "case x in x@(y)) ;; esac" + case_statement [0,24] "case x in x@(y)) ;; esac" + (case) [0,4] "case" + word [5,6] "x" + (in) [7,9] "in" + case_item [10,19] "x@(y)) ;;" + ERROR [10,15] "x@(y)" + ()) [15,16] ")" + (;;) [17,19] ";;" + (esac) [20,24] "esac" +=== +@match: case x in *([0-9])([0-9])) ;; esac +case x in *([0-9])([0-9])) ;; esac +=== +@match: case x in *"s"*) ;; esac +case x in *"s"*) ;; esac +=== +@match: case x in *${x}*) ;; esac +case x in *${x}*) ;; esac +=== +@match: case x in ${x}) ;; esac +case x in ${x}) ;; esac +=== +@match: case x in ?(a)) ;; esac +case x in ?(a)) ;; esac +=== +@match: case x in !(a)) ;; esac +case x in !(a)) ;; esac +=== +@match: case x in a\ b) ;; esac +case x in a\ b) ;; esac +=== +@match: case $x in foo_bar) ;; esac +case $x in foo_bar) ;; esac +=== +@match: case $x in word1) ;; esac +case $x in word1) ;; esac +=== +@match: case $x in abc1d) ;; esac +case $x in abc1d) ;; esac +=== +@match: case $x in v2.0) ;; esac +case $x in v2.0) ;; esac +=== +@match: case $x in a1) ;; esac +case $x in a1) ;; esac +=== +@match: case $x in a_) ;; esac +case $x in a_) ;; esac +=== +@match: case $x in A1) ;; esac +case $x in A1) ;; esac +=== +@match: case $x in a-b) ;; esac +case $x in a-b) ;; esac +=== +@match: case $x in a-bc) ;; esac +case $x in a-bc) ;; esac +=== +@match: case $x in ab-c) ;; esac +case $x in ab-c) ;; esac +=== +@match: case $x in a-b-c) ;; esac +case $x in a-b-c) ;; esac +=== +@match: case $x in cmake_modules) ;; esac +case $x in cmake_modules) ;; esac +=== +@match: case $x in a-b|y) ;; esac +case $x in a-b|y) ;; esac +=== +@match: case $x in abc|y) ;; esac +case $x in abc|y) ;; esac +=== +@match: case $x in abc | y) ;; esac +case $x in abc | y) ;; esac +=== +@match: case $x in -x-) ;; esac +case $x in -x-) ;; esac +=== +@match: case $x in -x-y) ;; esac +case $x in -x-y) ;; esac +=== +@known-diff case-rejected-group: case $x in *.(a|b)) ;; esac +case $x in *.(a|b)) ;; esac +--- +hasError: true +program [0,27] "case $x in *.(a|b)) ;; esac" + case_statement [0,27] "case $x in *.(a|b)) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,22] "*.(a|b)) ;;" + ERROR [11,18] "*.(a|b)" + ()) [18,19] ")" + (;;) [20,22] ";;" + (esac) [23,27] "esac" +=== +@known-diff case-rejected-group: case $x in x.(y)) ;; esac +case $x in x.(y)) ;; esac +--- +hasError: true +program [0,25] "case $x in x.(y)) ;; esac" + case_statement [0,25] "case $x in x.(y)) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,20] "x.(y)) ;;" + ERROR [11,16] "x.(y)" + ()) [16,17] ")" + (;;) [18,20] ";;" + (esac) [21,25] "esac" +=== +@known-diff case-pattern-after-continuation: case $x in cont\ued) ;; esac +case $x in cont\ +ued) ;; esac +--- +hasError: false +program [0,29] "case $x in cont\\\nued) ;; esac" + case_statement [0,29] "case $x in cont\\\nued) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,24] "cont\\\nued) ;;" + word [11,20] "cont\\\nued" + ()) [20,21] ")" + (;;) [22,24] ";;" + (esac) [25,29] "esac" +=== +@known-diff case-pattern-after-continuation: case $x in a\1) ;; esac +case $x in a\ +1) ;; esac +--- +hasError: false +program [0,24] "case $x in a\\\n1) ;; esac" + case_statement [0,24] "case $x in a\\\n1) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,19] "a\\\n1) ;;" + word [11,15] "a\\\n1" + ()) [15,16] ")" + (;;) [17,19] ";;" + (esac) [20,24] "esac" +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/expansions.txt b/packages/tree-sitter-bash/test/fixtures/differential/expansions.txt new file mode 100644 index 0000000000..e7394998a7 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/expansions.txt @@ -0,0 +1,599 @@ +@match: for f in a b c; do echo "$f"; done +for f in a b c; do echo "$f"; done +=== +@match: for f; do echo "$f"; done +for f; do echo "$f"; done +=== +@match: select opt in a b; do echo $opt; done +select opt in a b; do echo $opt; done +=== +@match: a[i+1]=$y +a[i+1]=$y +=== +@match: echo ${arr[1]} ${arr[@]} +echo ${arr[1]} ${arr[@]} +=== +@match: echo $'a\nb\t' +echo $'a\nb\t' +=== +@match: echo $"hello $USER" +echo $"hello $USER" +=== +@match: echo ${x##*/} ${x%%.*} +echo ${x##*/} ${x%%.*} +=== +@match: echo ${x/y/z} ${x//a/b} +echo ${x/y/z} ${x//a/b} +=== +@match: echo ${x^} ${x^^} ${x,} ${x,,} +echo ${x^} ${x^^} ${x,} ${x,,} +=== +@match: echo ${x@Q} +echo ${x@Q} +=== +@match: echo ${v:1:2} ${v:-default} +echo ${v:1:2} ${v:-default} +=== +@match: f() { arr=(1 2); echo ${arr[0]}; } +f() { arr=(1 2); echo ${arr[0]}; } +=== +@match: for i in $(seq 3); do echo $i; done +for i in $(seq 3); do echo $i; done +=== +@match: echo ${a[$i]} ${a[i+1]} +echo ${a[$i]} ${a[i+1]} +=== +@match: echo ${!prefix*} ${!name@} +echo ${!prefix*} ${!name@} +=== +@match: echo ${x##a b} +echo ${x##a b} +=== +@match: echo ${x#"pat"} +echo ${x#"pat"} +=== +@match: echo ${x/a b/c} +echo ${x/a b/c} +=== +@match: echo ${x/a/$v} +echo ${x/a/$v} +=== +@match: echo ${x/pat/rep/extra} +echo ${x/pat/rep/extra} +=== +@match: echo ${x:?msg} ${x:+alt} ${x:-d e f} +echo ${x:?msg} ${x:+alt} ${x:-d e f} +=== +@match: echo ${v:-1} ${v:=2x} ${v:-"d e"} +echo ${v:-1} ${v:=2x} ${v:-"d e"} +=== +@match: echo ${arr[@]:1:2} +echo ${arr[@]:1:2} +=== +@known-diff recovery-partial-nodes: echo ${s//"/x} +echo ${s//"/x} +--- +hasError: true +program [0,14] "echo ${s//\"/x}" + command [0,14] "echo ${s//\"/x}" + command_name [0,4] "echo" + word [0,4] "echo" + expansion [5,14] "${s//\"/x}" + (${) [5,7] "${" + variable_name [7,8] "s" + (//) [8,10] "//" + string [10,14] "\"/x}" + (") [10,11] "\"" + string_content [11,14] "/x}" +=== +@match: x=$"v" +x=$"v" +=== +@match: for f in $"a"; do x; done +for f in $"a"; do x; done +=== +@match: echo a$"b" +echo a$"b" +=== +@match: echo $"a""b" +echo $"a""b" +=== +@match: echo x=$"v" +echo x=$"v" +=== +@match: echo $0 ${0} $1 ${10} +echo $0 ${0} $1 ${10} +=== +@match: echo ${x: -5} +echo ${x: -5} +=== +@match: echo ${x: -5:2} +echo ${x: -5:2} +=== +@known-diff recovery-partial-nodes: ${x@} +${x@} +--- +hasError: true +program [0,5] "${x@}" + command [0,5] "${x@}" + command_name [0,5] "${x@}" + expansion [0,5] "${x@}" + (${) [0,2] "${" + variable_name [2,3] "x" + word [3,4] "@" + (}) [4,5] "}" +=== +@match: echo ${x[1]} +echo ${x[1]} +=== +@match: echo "hello $NAME" 'raw' +echo "hello $NAME" 'raw' +=== +@match: echo `hostname` $(date) +echo `hostname` $(date) +=== +@match: echo ${v:-d} ${x} $1 $@ +echo ${v:-d} ${x} $1 $@ +=== +@match: echo $HOME +echo $HOME +=== +@match: echo a"b"c'd'$e${f}g +echo a"b"c'd'$e${f}g +=== +@match: echo $v +echo $v +=== +@match: echo "${a} middle ${b}" +echo "${a} middle ${b}" +=== +@match: echo "$@" +echo "$@" +=== +@match: echo $(a $(b c)) +echo $(a $(b c)) +=== +@match: echo ${#v} +echo ${#v} +=== +@match: echo ${v:=word} +echo ${v:=word} +=== +@known-diff empty-command-substitution: echo $( ) +echo $( ) +--- +hasError: false +program [0,9] "echo $( )" + command [0,9] "echo $( )" + command_name [0,4] "echo" + word [0,4] "echo" + command_substitution [5,9] "$( )" + ($() [5,7] "$(" + ()) [8,9] ")" +=== +@known-diff empty-backtick: echo `` +echo `` +--- +hasError: false +program [0,7] "echo ``" + command [0,7] "echo ``" + command_name [0,4] "echo" + word [0,4] "echo" + command_substitution [5,7] "``" + (`) [5,6] "`" + (`) [6,7] "`" +=== +@match: echo $ +echo $ +=== +@match: echo "a\$b" +echo "a\$b" +=== +@known-diff string-content-newlines: echo "line1 +echo "line1 +line2" +--- +hasError: false +program [0,18] "echo \"line1\nline2\"" + command [0,18] "echo \"line1\nline2\"" + command_name [0,4] "echo" + word [0,4] "echo" + string [5,18] "\"line1\nline2\"" + (") [5,6] "\"" + string_content [6,17] "line1\nline2" + (") [17,18] "\"" +=== +@match: echo ${v:-$(cmd)} +echo ${v:-$(cmd)} +=== +@match: echo ${a[0]} +echo ${a[0]} +=== +@match: echo ${} +echo ${} +=== +@match: echo ${#} +echo ${#} +=== +@match: echo ${var1#*#} +echo ${var1#*#} +=== +@match: echo ${!abc} +echo ${!abc} +=== +@match: echo ${abc:- } +echo ${abc:- } +=== +@match: echo ${abc: +echo ${abc: +} +=== +@match: echo ${abc,?} +echo ${abc,?} +=== +@match: echo ${abc^^b} +echo ${abc^^b} +=== +@match: echo ${abc@U} +echo ${abc@U} +=== +@match: echo ${abc:- -quiet} +echo ${abc:- -quiet} +=== +@match: A="${B[0]# }" +A="${B[0]# }" +=== +@match: C="${D/#* -E /}" +C="${D/#* -E /}" +=== +@match: F="${G%% *}" +F="${G%% *}" +=== +@match: H="${I#*;}" +H="${I#*;}" +=== +@match: R="${S%()}" +R="${S%()}" +=== +@match: F="${#!}" +F="${#!}" +=== +@match: G=${H,,[I]} +G=${H,,[I]} +=== +@match: L="${M/'N'*/O}" +L="${M/'N'*/O}" +=== +@match: A=${B//:;;/$'\n'} +A=${B//:;;/$'\n'} +=== +@match: C=${D/;\ *;|} +C=${D/;\ *;|} +=== +@match: MOFILES=${LINGUAS// /.po }.po +MOFILES=${LINGUAS// /.po }.po +=== +@match: MY_P="${PN/aspell/aspell"${ASPELL_VERSION}"}" +MY_P="${PN/aspell/aspell"${ASPELL_VERSION}"}" +=== +@match: pyc=${pyc//*\/} +pyc=${pyc//*\/} +=== +@match: ${pv/\.} +${pv/\.} +=== +@match: ${new_test_cp//"${old_ver_cp}"/} +${new_test_cp//"${old_ver_cp}"/} +=== +@known-diff expansion-replacement-escaped-backslash: ${allarchives// /\\|} +${allarchives// /\\|} +--- +hasError: false +program [0,21] "${allarchives// /\\\\|}" + command [0,21] "${allarchives// /\\\\|}" + command_name [0,21] "${allarchives// /\\\\|}" + expansion [0,21] "${allarchives// /\\\\|}" + (${) [0,2] "${" + variable_name [2,13] "allarchives" + (//) [13,15] "//" + regex [15,16] " " + (/) [16,17] "/" + word [17,20] "\\\\|" + (}) [20,21] "}" +=== +@match: ${!#} +${!#} +=== +@known-diff expansion-bang-hash-special: ${!# } +${!# } +--- +hasError: true +program [0,6] "${!# }" + command [0,6] "${!# }" + command_name [0,6] "${!# }" + expansion [0,6] "${!# }" + (${) [0,2] "${" + (!) [2,3] "!" + special_variable_name [3,4] "#" + word [4,5] " " + (}) [5,6] "}" +=== +@match: ${!##} +${!##} +=== +@known-diff expansion-bang-hash-special: ${!## } +${!## } +--- +hasError: true +program [0,7] "${!## }" + command [0,7] "${!## }" + command_name [0,7] "${!## }" + expansion [0,7] "${!## }" + (${) [0,2] "${" + (!) [2,3] "!" + (#) [3,4] "#" + special_variable_name [4,5] "#" + word [5,6] " " + (}) [6,7] "}" +=== +@known-diff expansion-bang-hash-special: ${!##/} +${!##/} +--- +hasError: false +program [0,7] "${!##/}" + command [0,7] "${!##/}" + command_name [0,7] "${!##/}" + expansion [0,7] "${!##/}" + (${) [0,2] "${" + (!) [2,3] "!" + (#) [3,4] "#" + special_variable_name [4,5] "#" + (/) [5,6] "/" + (}) [6,7] "}" +=== +@match: echo "${kw}? ( ${cond:+${cond}? (} ${baseuri}-${ver}-${kw... +echo "${kw}? ( ${cond:+${cond}? (} ${baseuri}-${ver}-${kw}.${suff} ${cond:+) })" +=== +@match: ${completions[*]} +${completions[*]} +=== +@known-diff expansion-equals-operator: ${=1} +${=1} +--- +hasError: false +program [0,5] "${=1}" + command [0,5] "${=1}" + command_name [0,5] "${=1}" + expansion [0,5] "${=1}" + (${) [0,2] "${" + (=) [2,3] "=" + word [3,4] "1" + (}) [4,5] "}" +=== +@match: ${2?} +${2?} +=== +@match: ${p_key#*=} +${p_key#*=} +=== +@match: ${B[0]# } +${B[0]# } +=== +@match: ${to_enables[0]##*/} +${to_enables[0]##*/} +=== +@match: exec "${0#-}" --rcfile "${BASH_IT_BASHRC:-${HOME?}/.bashrc}" +exec "${0#-}" --rcfile "${BASH_IT_BASHRC:-${HOME?}/.bashrc}" +=== +@match: recho "TDEFAULTS = ${selvecs:+-DSELECT_VECS=\"$selvecs\"}" +recho "TDEFAULTS = ${selvecs:+-DSELECT_VECS=\"$selvecs\"}" +=== +@match: local msg="${2:-command '$1' does not exist}" +local msg="${2:-command '$1' does not exist}" +=== +@match: ${cdir:+#} +${cdir:+#} +=== +@match: ${dict_langs:+;} +${dict_langs:+;} +=== +@match: ${UTIL_LINUX_LIBC[@]/%/? ( sys-apps/util-linux )} +${UTIL_LINUX_LIBC[@]/%/? ( sys-apps/util-linux )} +=== +@match: ${id}${2+ ${2}} +${id}${2+ ${2}} +=== +@match: ${BRANDING_GCC_PKGVERSION/(/(Gentoo ${PVR}${extvers}, } +${BRANDING_GCC_PKGVERSION/(/(Gentoo ${PVR}${extvers}, } +=== +@match: BASH_IT_GIT_URL="${BASH_IT_GIT_URL/git@/https://}" +BASH_IT_GIT_URL="${BASH_IT_GIT_URL/git@/https://}" +=== +@known-diff number-base-with-expansion: 10#${command_start##*.} +10#${command_start##*.} +--- +hasError: false +program [0,23] "10#${command_start##*.}" + command [0,23] "10#${command_start##*.}" + command_name [0,23] "10#${command_start##*.}" + concatenation [0,23] "10#${command_start##*.}" + word [0,3] "10#" + expansion [3,23] "${command_start##*.}" + (${) [3,5] "${" + variable_name [5,18] "command_start" + (##) [18,20] "##" + regex [20,22] "*." + (}) [22,23] "}" +=== +@match: echo ${LIB_DEPEND//\[static-libs(+)]} +echo ${LIB_DEPEND//\[static-libs(+)]} +=== +@match: ${ALL_LLVM_TARGETS[@]/%/(-)?} +${ALL_LLVM_TARGETS[@]/%/(-)?} +=== +@match: filterdiff -p1 ${paths[@]/#/-i } +filterdiff -p1 ${paths[@]/#/-i } +=== +@match: ${cflags//-O? /$(get-flag O) } +${cflags//-O? /$(get-flag O) } +=== +@match: curf="${f%'-roff2html'*}.html" +curf="${f%'-roff2html'*}.html" +=== +@match: reff="${f/'-roff2html'*/'-ref'}.html" +reff="${f/'-roff2html'*/'-ref'}.html" +=== +@match: echo "s/$/'/" +echo "s/$/'/" +=== +@match: echo "s$" +echo "s$" +=== +@match: echo $'Here\'s Johnny!\r\n' +echo $'Here\'s Johnny!\r\n' +=== +@known-diff expansion-default-array: printf " %-9s" "${seq0:-(default)}" +printf " %-9s" "${seq0:-(default)}" +--- +hasError: false +program [0,36] "printf \" %-9s\" \"${seq0:-(default)}\"" + command [0,36] "printf \" %-9s\" \"${seq0:-(default)}\"" + command_name [0,6] "printf" + word [0,6] "printf" + string [7,15] "\" %-9s\"" + (") [7,8] "\"" + string_content [8,14] " %-9s" + (") [14,15] "\"" + string [16,36] "\"${seq0:-(default)}\"" + (") [16,17] "\"" + expansion [17,35] "${seq0:-(default)}" + (${) [17,19] "${" + variable_name [19,23] "seq0" + (:-) [23,25] ":-" + word [25,34] "(default)" + (}) [34,35] "}" + (") [35,36] "\"" +=== +@known-diff empty-backtick: main() { +main() { + local foo="asd"` + `"fgh" +} +--- +hasError: false +program [0,46] "main() {\n local foo=\"asd\"`\n ..." + function_definition [0,46] "main() {\n local foo=\"asd\"`\n ..." + word [0,4] "main" + (() [4,5] "(" + ()) [5,6] ")" + compound_statement [7,46] "{\n local foo=\"asd\"`\n `\"fgh\"\n}" + ({) [7,8] "{" + declaration_command [13,44] "local foo=\"asd\"`\n `\"fgh\"" + (local) [13,18] "local" + variable_assignment [19,44] "foo=\"asd\"`\n `\"fgh\"" + variable_name [19,22] "foo" + (=) [22,23] "=" + concatenation [23,44] "\"asd\"`\n `\"fgh\"" + string [23,28] "\"asd\"" + (") [23,24] "\"" + string_content [24,27] "asd" + (") [27,28] "\"" + command_substitution [28,39] "`\n `" + (`) [28,29] "`" + (`) [38,39] "`" + string [39,44] "\"fgh\"" + (") [39,40] "\"" + string_content [40,43] "fgh" + (") [43,44] "\"" + (}) [45,46] "}" +=== +@known-diff dollar-backtick-substitution: $(eval echo $`echo ${foo}`) +$(eval echo $`echo ${foo}`) +--- +hasError: false +program [0,27] "$(eval echo $`echo ${foo}`)" + command [0,27] "$(eval echo $`echo ${foo}`)" + command_name [0,27] "$(eval echo $`echo ${foo}`)" + command_substitution [0,27] "$(eval echo $`echo ${foo}`)" + ($() [0,2] "$(" + command [2,26] "eval echo $`echo ${foo}`" + command_name [2,6] "eval" + word [2,6] "eval" + word [7,11] "echo" + concatenation [12,26] "$`echo ${foo}`" + ($) [12,13] "$" + command_substitution [13,26] "`echo ${foo}`" + (`) [13,14] "`" + command [14,25] "echo ${foo}" + command_name [14,18] "echo" + word [14,18] "echo" + expansion [19,25] "${foo}" + (${) [19,21] "${" + variable_name [21,24] "foo" + (}) [24,25] "}" + (`) [25,26] "`" + ()) [26,27] ")" +=== +@match: echo `echo otherword`word +echo `echo otherword`word +=== +@match: echo word`echo otherword` +echo word`echo otherword` +=== +@match: assert() +assert() + if ! $1; then + return 1 + fi +=== +@match: echo " ${x}" +echo " ${x}" +=== +@match: echo "a ${x} b" +echo "a ${x} b" +=== +@match: echo "$x " +echo "$x " +=== +@match: echo "${x} " +echo "${x} " +=== +@match: echo ${#@} +echo ${#@} +=== +@match: echo ${#*} +echo ${#*} +=== +@match: echo ${#-} +echo ${#-} +=== +@match: echo ${#x[@]} +echo ${#x[@]} +=== +@match: echo ${x/'abc'/y} +echo ${x/'abc'/y} +=== +@match: echo ${x/"abc"/y} +echo ${x/"abc"/y} +=== +@match: echo ${x/'a'*'b'/y} +echo ${x/'a'*'b'/y} +=== +@match: echo ${x##'a'} +echo ${x##'a'} +=== +@match: echo ${x%%a b} +echo ${x%%a b} +=== +@match: echo "${x} $(y)" +echo "${x} $(y)" +=== +@match: echo " $(y)" +echo " $(y)" +=== +@match: echo " $x" +echo " $x" +=== +@match: echo "${x} $y" +echo "${x} $y" +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt b/packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt new file mode 100644 index 0000000000..1ac604b146 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt @@ -0,0 +1,597 @@ +@match: echo $((x << 2 | 1)) +echo $((x << 2 | 1)) +=== +@known-diff heredoc-content-chunks: foo() { cat < out +cat < out +body +EOF +--- +hasError: false +program [0,24] "cat < out\nbody\nEOF" + redirected_statement [0,24] "cat < out\nbody\nEOF" + command [0,3] "cat" + command_name [0,3] "cat" + word [0,3] "cat" + heredoc_redirect [4,24] "< out\nbody\nEOF" + (<<) [4,6] "<<" + heredoc_start [6,9] "EOF" + file_redirect [10,15] "> out" + (>) [10,11] ">" + word [12,15] "out" + heredoc_body [16,21] "body\n" + heredoc_content [16,21] "body\n" + heredoc_end [21,24] "EOF" +=== +@known-diff heredoc-tail-statements: cat < out +cmd <<< x > out +=== +@known-diff heredoc-multiple-on-line: cat <> 10 & 11 | 12 ^ 13)) +a=$((6 % 7 ** 8 << 9 >> 10 & 11 | 12 ^ 13)) +=== +@known-diff heredoc-content-chunks: cat < out +foo() { x; } > out +=== +@match: if a; then b; fi > out +if a; then b; fi > out +=== +@match: cat > $"out" +cat > $"out" +=== +@match: diff <(cmd) >(other) +diff <(cmd) >(other) +=== +@match: cmd > out 2>&1 +cmd > out 2>&1 +=== +@match: cmd &>> log +cmd &>> log +=== +@known-diff redirect-readwrite: exec 3<>file +exec 3<>file +--- +hasError: false +program [0,12] "exec 3<>file" + redirected_statement [0,12] "exec 3<>file" + command [0,4] "exec" + command_name [0,4] "exec" + word [0,4] "exec" + file_redirect [5,12] "3<>file" + file_descriptor [5,6] "3" + (<>) [6,8] "<>" + word [8,12] "file" +=== +@match: > out +> out +=== +@match: cmd 2>/dev/null +cmd 2>/dev/null +=== +@match: cmd >&- +cmd >&- +=== +@match: cmd >| file +cmd >| file +=== +@known-diff redirect-readwrite: cmd <>file +cmd <>file +--- +hasError: false +program [0,10] "cmd <>file" + redirected_statement [0,10] "cmd <>file" + command [0,3] "cmd" + command_name [0,3] "cmd" + word [0,3] "cmd" + file_redirect [4,10] "<>file" + (<>) [4,6] "<>" + word [6,10] "file" +=== +@match: cmd < in +cmd < in +=== +@match: cmd <&0- +cmd <&0- +=== +@match: cmd > out +cmd > out +=== +@match: cmd 2>&1 >/dev/null +cmd 2>&1 >/dev/null +=== +@match: > a cmd x +> a cmd x +=== +@match: (ls) > out +(ls) > out +=== +@match: ! ls > out +! ls > out +=== +@match: cmd > out arg +cmd > out arg +=== +@match: 2>&1 whoami +2>&1 whoami +=== +@match: echo "foobar" >&2 +echo "foobar" >&2 +=== +@match: exec {VIRTWL[0]} {VIRTWL[1]} <&- >&- +exec {VIRTWL[0]} {VIRTWL[1]} <&- >&- +=== +@match: exec {VIRTWL[0]}<&- {VIRTWL[1]}>&- +exec {VIRTWL[0]}<&- {VIRTWL[1]}>&- +=== +@match: x &- +exec {a[0]}<&- {b[1]}>&- +=== +@match: exec {a[0]}<&- +exec {a[0]}<&- +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/statements.txt b/packages/tree-sitter-bash/test/fixtures/differential/statements.txt new file mode 100644 index 0000000000..24c53c5ea2 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/statements.txt @@ -0,0 +1,391 @@ +@match: if true; then echo yes; fi +if true; then echo yes; fi +=== +@match: if a; then b; elif c; then d; else e; fi +if a; then b; elif c; then d; else e; fi +=== +@match: if a +if a +then + b +fi +=== +@match: until x; do y; done +until x; do y; done +=== +@match: foo() { echo hi; } +foo() { echo hi; } +=== +@match: function bar { echo hi; } +function bar { echo hi; } +=== +@match: function baz() { echo hi; return 0; } +function baz() { echo hi; return 0; } +=== +@match: foo() ( echo sub ) +foo() ( echo sub ) +=== +@match: { echo a; echo b; } +{ echo a; echo b; } +=== +@match: coproc myjob { echo hi; } +coproc myjob { echo hi; } +=== +@match: echo if for while +echo if for while +=== +@match: arr=(a b "c d") +arr=(a b "c d") +=== +@match: arr[0]=x +arr[0]=x +=== +@match: export FOO=bar BAZ +export FOO=bar BAZ +=== +@match: declare -r x=1 +declare -r x=1 +=== +@match: local y +local y +=== +@match: readonly z=2 +readonly z=2 +=== +@match: unset a b +unset a b +=== +@match: echo {1..10} +echo {1..10} +=== +@match: echo {1..10..2} +echo {1..10..2} +=== +@match: echo {a..z} +echo {a..z} +=== +@match: echo a{b,c}d +echo a{b,c}d +=== +@match: time ls -la +time ls -la +=== +@match: ls +ls +echo b +=== +@match: echo [ x ] +echo [ x ] +=== +@match: if [ -f f ]; then x; fi +if [ -f f ]; then x; fi +=== +@match: foo() if a; then b; fi +foo() if a; then b; fi +=== +@match: if a; then b; fi | grep x +if a; then b; fi | grep x +=== +@match: if ! grep -q x f; then echo missing; fi +if ! grep -q x f; then echo missing; fi +=== +@match: x[1]+=2 +x[1]+=2 +=== +@match: until grep -q done log; do sleep 1; done & +until grep -q done log; do sleep 1; done & +=== +@match: ( (ls) ) +( (ls) ) +=== +@match: {ls;} +{ls;} +=== +@match: fi +fi +=== +@match: done +done +=== +@match: } +} +=== +@match: echo a{1..3}b +echo a{1..3}b +=== +@match: x={1..5} +x={1..5} +=== +@match: while x; do done +while x; do done +=== +@match: echo {a..z..2} +echo {a..z..2} +=== +@match: echo {10..1} +echo {10..1} +=== +@match: echo {-5..5} +echo {-5..5} +=== +@match: select x in; do y; done +select x in; do y; done +=== +@match: function f() if x; then y; fi +function f() if x; then y; fi +=== +@match: f () { x; } +f () { x; } +=== +@match: if a && b; then c; fi +if a && b; then c; fi +=== +@match: foo (ls) +foo (ls) +=== +@match: for f in a b +for f in a b +do x +done +=== +@match: while a; b; do x; done +while a; b; do x; done +=== +@match: if a; then b; fi && c +if a; then b; fi && c +=== +@match: if if a; then b; fi; then c; fi +if if a; then b; fi; then c; fi +=== +@match: x=1 if a; then b; fi +x=1 if a; then b; fi +=== +@match: echo hi | if a; then b; fi +echo hi | if a; then b; fi +=== +@match: ! while x; do y; done +! while x; do y; done +=== +@match: ! [ -f x ] +! [ -f x ] +=== +@match: x+=(c d) +x+=(c d) +=== +@match: declare -A m +declare -A m +=== +@match: {1..3} +{1..3} +=== +@match: echo {1..3} +echo {1..3} +=== +@match: if echo then; then x; fi +if echo then; then x; fi +=== +@match: while echo do; do x; done +while echo do; do x; done +=== +@match: for f in a; do echo done; done +for f in a; do echo done; done +=== +@match: { echo }; } +{ echo }; } +=== +@match: [1] +[1] +=== +@match: echo [1] a[2]b +echo [1] a[2]b +=== +@match: echo a[-5]b +echo a[-5]b +=== +@match: if a; then b; elif c; then d +if a; then b; elif c; then d +else e +fi +=== +@match: ls -la +ls -la +=== +@match: FOO=bar env | grep FOO && echo ok || echo fail +FOO=bar env | grep FOO && echo ok || echo fail +=== +@match: ! grep -q x file +! grep -q x file +=== +@match: # a comment +# a comment +echo hi # trailing +=== +@match: (ls) +(ls) +=== +@match: FOO=bar ls +FOO=bar ls +=== +@known-diff trailing-connector: ls && +ls && +--- +hasError: true +program [0,5] "ls &&" + list [0,5] "ls &&" + command [0,2] "ls" + command_name [0,2] "ls" + word [0,2] "ls" + (&&) [3,5] "&&" +=== +@match: a & b & +a & b & +=== +@match: echo 123 45.6 +echo 123 45.6 +=== +@match: VAR=value +VAR=value +=== +@match: ls; echo a +ls; echo a +echo b +=== +@match: FOO=1 BAR=2 cmd arg +FOO=1 BAR=2 cmd arg +=== +@match: ! a | b +! a | b +=== +@match: echo "" +echo "" +=== +@match: VAR= +VAR= +=== +@match: a|b|c +a|b|c +=== +@match: ls ; +ls ; +=== +@match: echo hi;# c +echo hi;# c +ls +=== +@match: A=1 B=2 +A=1 B=2 +=== +@match: a && b && c +a && b && c +=== +@match: echo x\ +echo x\ +y +=== +@match: echo {a,b} +echo {a,b} +=== +@known-diff trailing-connector: ls | +ls | +--- +hasError: true +program [0,4] "ls |" + pipeline [0,4] "ls |" + command [0,2] "ls" + command_name [0,2] "ls" + word [0,2] "ls" + (|) [3,4] "|" +=== +@match: time ls +time ls +=== +@match: FOO="a b" cmd +FOO="a b" cmd +=== +@match: FOO=a"b"c +FOO=a"b"c +=== +@known-diff escaped-space-argument: echo 1 \ 2 \ 3 +echo 1 \ 2 \ 3 +--- +hasError: false +program [0,14] "echo 1 \\ 2 \\ 3" + command [0,14] "echo 1 \\ 2 \\ 3" + command_name [0,4] "echo" + word [0,4] "echo" + number [5,6] "1" + word [7,10] "\\ 2" + word [11,14] "\\ 3" +=== +@known-diff expansion-equals-operator: for c in ${=1}; do +for c in ${=1}; do + echo c +done +--- +hasError: false +program [0,31] "for c in ${=1}; do\n\techo c\ndone" + for_statement [0,31] "for c in ${=1}; do\n\techo c\ndone" + (for) [0,3] "for" + variable_name [4,5] "c" + (in) [6,8] "in" + expansion [9,14] "${=1}" + (${) [9,11] "${" + (=) [11,12] "=" + word [12,13] "1" + (}) [13,14] "}" + (;) [14,15] ";" + do_group [16,31] "do\n\techo c\ndone" + (do) [16,18] "do" + command [20,26] "echo c" + command_name [20,24] "echo" + word [20,24] "echo" + word [25,26] "c" + (done) [27,31] "done" +=== +@match: foo::bar() { +foo::bar() { + echo what +} +=== +@match: echo "a " +echo "a " +=== +@match: echo 你好 && ls +echo 你好 && ls +=== +@match: echo "🎉 done" | grep 🎉 && echo ✓ +echo "🎉 done" | grep 🎉 && echo ✓ +=== +@known-diff nonascii-identifier-assignment: 变量=值 echo $变量 +变量=值 echo $变量 +--- +hasError: false +program [0,13] "变量=值 echo $变量" + command [0,13] "变量=值 echo $变量" + command_name [0,4] "变量=值" + word [0,4] "变量=值" + word [5,9] "echo" + concatenation [10,13] "$变量" + ($) [10,11] "$" + word [11,13] "变量" +=== +@known-diff escaped-space-argument: echo a\b +echo a\ b +--- +hasError: false +program [0,9] "echo a\\\tb" + command [0,9] "echo a\\\tb" + command_name [0,4] "echo" + word [0,4] "echo" + word [5,9] "a\\\tb" +=== +@known-diff escaped-space-argument: echo \x +echo \ x +--- +hasError: false +program [0,8] "echo \\\tx" + command [0,8] "echo \\\tx" + command_name [0,4] "echo" + word [0,4] "echo" + word [5,8] "\\\tx" +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/test-command.txt b/packages/tree-sitter-bash/test/fixtures/differential/test-command.txt new file mode 100644 index 0000000000..02b22149ef --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/test-command.txt @@ -0,0 +1,650 @@ +@match: [[ -f file.txt ]] +[[ -f file.txt ]] +=== +@match: [[ $x == "foo" ]] +[[ $x == "foo" ]] +=== +@match: [[ $x =~ ^ab+c$ ]] +[[ $x =~ ^ab+c$ ]] +=== +@match: [[ -n $s && -d /tmp ]] +[[ -n $s && -d /tmp ]] +=== +@match: [[ a < b || ! -e f ]] +[[ a < b || ! -e f ]] +=== +@match: [ -f file ] +[ -f file ] +=== +@match: [ "$a" = "b" ] +[ "$a" = "b" ] +=== +@match: ! [[ -f x ]] +! [[ -f x ]] +=== +@match: [ -f file +[ -f file +=== +@match: [ x ] +[ x ] +=== +@match: while [[ -n $s ]]; do s=; done +while [[ -n $s ]]; do s=; done +=== +@match: foo() [[ -f x ]] +foo() [[ -f x ]] +=== +@match: [[ -z $s ]] +[[ -z $s ]] +=== +@match: [[ $a -eq 3 ]] +[[ $a -eq 3 ]] +=== +@match: [[ $a -nt $b ]] +[[ $a -nt $b ]] +=== +@match: [[ "x" ]] +[[ "x" ]] +=== +@match: [[ ]] +[[ ]] +=== +@match: [[ a = b*c ]] +[[ a = b*c ]] +=== +@match: [[ a = bc ]] +[[ a = bc ]] +=== +@match: [[ $x == b*c ]] +[[ $x == b*c ]] +=== +@match: [[ $x == a"b" ]] +[[ $x == a"b" ]] +=== +@match: [[ $x =~ ^a.*b$ ]] +[[ $x =~ ^a.*b$ ]] +=== +@match: [[ a != b ]] +[[ a != b ]] +=== +@match: [[ (a == b) && c ]] +[[ (a == b) && c ]] +=== +@match: [[ -v var ]] +[[ -v var ]] +=== +@match: [[ -o opt ]] +[[ -o opt ]] +=== +@match: [[ $x -eq 1 && $y -ne 2 ]] +[[ $x -eq 1 && $y -ne 2 ]] +=== +@match: [[ $x == b ]] +[[ $x == b ]] +=== +@known-diff recovery-partial-nodes: [[ $x =~ ^a b$ ]] +[[ $x =~ ^a b$ ]] +--- +hasError: true +program [0,17] "[[ $x =~ ^a b$ ]]" + test_command [0,17] "[[ $x =~ ^a b$ ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$x =~ ^a" + simple_expansion [3,5] "$x" + ($) [3,4] "$" + variable_name [4,5] "x" + (=~) [6,8] "=~" + regex [9,11] "^a" + ERROR [12,15] "b$ " + (]]) [15,17] "]]" +=== +@match: [[ $x =~ a|b ]] +[[ $x =~ a|b ]] +=== +@match: [[ $x == b && $y == c ]] +[[ $x == b && $y == c ]] +=== +@match: [[ $x == 123 ]] +[[ $x == 123 ]] +=== +@match: [[ -t 1 ]] +[[ -t 1 ]] +=== +@match: [[ $x =~ $re ]] +[[ $x =~ $re ]] +=== +@match: [[ $x =~ "a b" ]] +[[ $x =~ "a b" ]] +=== +@match: [[ a=b ]] +[[ a=b ]] +=== +@match: [[ $a==b* ]] +[[ $a==b* ]] +=== +@match: [[ ( $a == x* ) ]] +[[ ( $a == x* ) ]] +=== +@match: [[ ( $a = x* ) ]] +[[ ( $a = x* ) ]] +=== +@match: [[ $x == b=c ]] +[[ $x == b=c ]] +=== +@match: [[ a == =b ]] +[[ a == =b ]] +=== +@match: [[ a = b=c ]] +[[ a = b=c ]] +=== +@match: [[ -f file +[[ -f file +=== +@known-diff recovery-partial-nodes: [[ $a +[[ $a +--- +hasError: true +program [0,5] "[[ $a" + test_command [0,5] "[[ $a" + ([[) [0,2] "[[" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (]]) [5,5] "" +=== +@match: [ 1 ] +[ 1 ] +=== +@match: [[ 1 ]] +[[ 1 ]] +=== +@match: [[ $f == foo*(.txt|.log) ]] +[[ $f == foo*(.txt|.log) ]] +=== +@match: [[ $f == *foo*(a) ]] +[[ $f == *foo*(a) ]] +=== +@match: [[ $f == [a-z]*(x) ]] +[[ $f == [a-z]*(x) ]] +=== +@known-diff test-extglob-rejected-group: [[ $f == x@(y|z)w ]] +[[ $f == x@(y|z)w ]] +--- +hasError: true +program [0,20] "[[ $f == x@(y|z)w ]]" + test_command [0,20] "[[ $f == x@(y|z)w ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$f == x@" + simple_expansion [3,5] "$f" + ($) [3,4] "$" + variable_name [4,5] "f" + (==) [6,8] "==" + word [9,11] "x@" + ERROR [11,18] "(y|z)w " + (]]) [18,20] "]]" +=== +@known-diff test-negative-decimal: [[ $n == -0.5 ]] +[[ $n == -0.5 ]] +--- +hasError: false +program [0,16] "[[ $n == -0.5 ]]" + test_command [0,16] "[[ $n == -0.5 ]]" + ([[) [0,2] "[[" + binary_expression [3,13] "$n == -0.5" + simple_expansion [3,5] "$n" + ($) [3,4] "$" + variable_name [4,5] "n" + (==) [6,8] "==" + extglob_pattern [9,13] "-0.5" + (]]) [14,16] "]]" +=== +@known-diff test-fused-operator-expansion: [[ -x$f && -d /tmp ]] +[[ -x$f && -d /tmp ]] +--- +hasError: false +program [0,21] "[[ -x$f && -d /tmp ]]" + test_command [0,21] "[[ -x$f && -d /tmp ]]" + ([[) [0,2] "[[" + binary_expression [3,18] "-x$f && -d /tmp" + concatenation [3,7] "-x$f" + word [3,5] "-x" + simple_expansion [5,7] "$f" + ($) [5,6] "$" + variable_name [6,7] "f" + (&&) [8,10] "&&" + unary_expression [11,18] "-d /tmp" + test_operator [11,13] "-d" + word [14,18] "/tmp" + (]]) [19,21] "]]" +=== +@known-diff test-escaped-pipe-pattern: [[ $a == foo\|bar ]] +[[ $a == foo\|bar ]] +--- +hasError: false +program [0,20] "[[ $a == foo\\|bar ]]" + test_command [0,20] "[[ $a == foo\\|bar ]]" + ([[) [0,2] "[[" + binary_expression [3,17] "$a == foo\\|bar" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + extglob_pattern [9,17] "foo\\|bar" + (]]) [18,20] "]]" +=== +@known-diff recovery-partial-nodes: [[ -f +[[ -f +--- +hasError: true +program [0,5] "[[ -f" + test_command [0,5] "[[ -f" + ([[) [0,2] "[[" + word [3,5] "-f" + (]]) [5,5] "" +=== +@known-diff recovery-partial-nodes: [[ a +[[ a +--- +hasError: true +program [0,4] "[[ a" + test_command [0,4] "[[ a" + ([[) [0,2] "[[" + word [3,4] "a" + (]]) [4,4] "" +=== +@known-diff recovery-partial-nodes: [[ a == b +[[ a == b +--- +hasError: true +program [0,9] "[[ a == b" + test_command [0,9] "[[ a == b" + ([[) [0,2] "[[" + binary_expression [3,9] "a == b" + word [3,4] "a" + (==) [5,7] "==" + word [8,9] "b" + (]]) [9,9] "" +=== +@known-diff recovery-partial-nodes: [[ a && +[[ a && +--- +hasError: true +program [0,7] "[[ a &&" + test_command [0,7] "[[ a &&" + ([[) [0,2] "[[" + binary_expression [3,7] "a &&" + word [3,4] "a" + (&&) [5,7] "&&" + (]]) [7,7] "" +=== +@match: [[ -f file +[[ -f file +=== +@match: [[ a = =b ]] +[[ a = =b ]] +=== +@match: [[ $a =~^x ]] +[[ $a =~^x ]] +=== +@match: [[ !x = y ]] +[[ !x = y ]] +=== +@match: [[ a =b ]] +[[ a =b ]] +=== +@match: [[ a!=b ]] +[[ a!=b ]] +=== +@match: [[ a ==b ]] +[[ a ==b ]] +=== +@match: [[ $a == foo ]] +[[ $a == foo ]] +=== +@match: [[ $a == x.y ]] +[[ $a == x.y ]] +=== +@match: [[ ( $a == x ) ]] +[[ ( $a == x ) ]] +=== +@match: [[ $a == x1 ]] +[[ $a == x1 ]] +=== +@match: [[ $a == foo- ]] +[[ $a == foo- ]] +=== +@match: [[ $a == x_ ]] +[[ $a == x_ ]] +=== +@match: [[ a == b*c=d ]] +[[ a == b*c=d ]] +=== +@match: [[ a == bc=d ]] +[[ a == bc=d ]] +=== +@match: [[ a == b?=e ]] +[[ a == b?=e ]] +=== +@match: [[ a == .x ]] +[[ a == .x ]] +=== +@match: [[ $a == a-b ]] +[[ $a == a-b ]] +=== +@match: [[ ((a)) == x ]] +[[ ((a)) == x ]] +=== +@match: [[ $a == -foo ]] +[[ $a == -foo ]] +=== +@match: [[ $a == -f ]] +[[ $a == -f ]] +=== +@match: [[ -foo == x ]] +[[ -foo == x ]] +=== +@match: [[ $a == -1 ]] +[[ $a == -1 ]] +=== +@match: [[ $a -eq -1 ]] +[[ $a -eq -1 ]] +=== +@match: [[ $a == +(!a) ]] +[[ $a == +(!a) ]] +=== +@match: [[ $a == ?(a|b) ]] +[[ $a == ?(a|b) ]] +=== +@match: [[ $a == !(x) ]] +[[ $a == !(x) ]] +=== +@match: [[ $a == b\*c ]] +[[ $a == b\*c ]] +=== +@match: [[ $ver == [0-9]* ]] +[[ $ver == [0-9]* ]] +=== +@match: [[ $a == [abc] ]] +[[ $a == [abc] ]] +=== +@match: [[ ( $a == [xyz] ) ]] +[[ ( $a == [xyz] ) ]] +=== +@match: [[ $a = [abc] ]] +[[ $a = [abc] ]] +=== +@match: [[ $a == [!a]* ]] +[[ $a == [!a]* ]] +=== +@match: [[ $a == foo\ bar ]] +[[ $a == foo\ bar ]] +=== +@known-diff test-paren-group-logical: [[ ((a) == x) && y ]] +[[ ((a) == x) && y ]] +--- +hasError: false +program [0,21] "[[ ((a) == x) && y ]]" + test_command [0,21] "[[ ((a) == x) && y ]]" + ([[) [0,2] "[[" + binary_expression [3,18] "((a) == x) && y" + parenthesized_expression [3,13] "((a) == x)" + (() [3,4] "(" + binary_expression [4,12] "(a) == x" + parenthesized_expression [4,7] "(a)" + (() [4,5] "(" + word [5,6] "a" + ()) [6,7] ")" + (==) [8,10] "==" + word [11,12] "x" + ()) [12,13] ")" + (&&) [14,16] "&&" + word [17,18] "y" + (]]) [19,21] "]]" +=== +@match: [ ! command -v go &>/dev/null ] && return +[ ! command -v go &>/dev/null ] && return +=== +@match: [[ ${f} != */@(default).vim ]] +[[ ${f} != */@(default).vim ]] +=== +@match: [[ $(LC_ALL=C $(tc-getCC) ${LDFLAGS} -Wl,--version 2>/dev... +[[ $(LC_ALL=C $(tc-getCC) ${LDFLAGS} -Wl,--version 2>/dev/null) != @(LLD|GNU\ ld)* ]] +=== +@match: [[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] +=== +@match: [[ ${SRC_URI} == */${a}* ]] +[[ ${SRC_URI} == */${a}* ]] +=== +@match: [[ a == *_@(LIB|SYMLINK) ]] +[[ a == *_@(LIB|SYMLINK) ]] +=== +@match: [[ "35d8b" =~ ^[0-9a-fA-F] ]] +[[ "35d8b" =~ ^[0-9a-fA-F] ]] +=== +@match: [[ $CMD =~ (^|;)update_terminal_cwd($|;) ]] +[[ $CMD =~ (^|;)update_terminal_cwd($|;) ]] +=== +@match: [[ ! " ${completions[*]} " =~ " $alias_cmd " ]] +[[ ! " ${completions[*]} " =~ " $alias_cmd " ]] +=== +@match: ! [[ "$a" =~ ^a|b\ *c|d$ ]] +! [[ "$a" =~ ^a|b\ *c|d$ ]] +=== +@match: [[ "$1" =~ ^${var}${var}*=..* ]] +[[ "$1" =~ ^${var}${var}*=..* ]] +=== +@match: [[ "$1" =~ ^\-${var}+ ]] +[[ "$1" =~ ^\-${var}+ ]] +=== +@match: [[ ${var1} == *${var2}* ]] +[[ ${var1} == *${var2}* ]] +=== +@match: [[ "$server" =~ [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]... +[[ "$server" =~ [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3} ]] +=== +@match: [[ -f ${x} && $(od -t x1 -N 4 "${x}") == *"7f 45 4c 46"* ]] +[[ -f ${x} && $(od -t x1 -N 4 "${x}") == *"7f 45 4c 46"* ]] +=== +@match: [[ " ${REPLACING_VERSIONS} " != *\ ${PVR}\ * ]] +[[ " ${REPLACING_VERSIONS} " != *\ ${PVR}\ * ]] +=== +@match: [[ ${file} == @(*${GENTOO_PATCH_NAME}.tar.xz|*.asc|*.sig) ]] +[[ ${file} == @(*${GENTOO_PATCH_NAME}.tar.xz|*.asc|*.sig) ]] +=== +@match: [[ $RUBY_TARGETS != *$( eselect ruby show | awk 'NR==2' |... +[[ $RUBY_TARGETS != *$( eselect ruby show | awk 'NR==2' | tr -d ' ' )* ]] +=== +@match: [[ " ${m[0]##*/}" =~ ^(\ ${skip_files[*]/%/.*|\\} ) ]] +[[ " ${m[0]##*/}" =~ ^(\ ${skip_files[*]/%/.*|\\} ) ]] +=== +@match: [[ ' boop xyz' =~ ' boop '(.*)$ ]] +[[ ' boop xyz' =~ ' boop '(.*)$ ]] +=== +@match: [[ $b = foo* ]] +[[ $b = foo* ]] +=== +@match: [[ a == \"+(?)\" ]] +[[ a == \"+(?)\" ]] +=== +@match: [[ ${PV} != $(sed -n -e 's/^Version: //p' "${ED}/usr/$(ge... +[[ ${PV} != $(sed -n -e 's/^Version: //p' "${ED}/usr/$(get_libdir)/pkgconfig/tss2-tcti-tabrmd.pc" || die) ]] +=== +@match: [[ "${MY_LOCALES}" != *en_US* || a != 2 ]] +[[ "${MY_LOCALES}" != *en_US* || a != 2 ]] +=== +@match: [[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \ +[[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \ + -x "${EROOT}/usr/bin/peardev" ]] +=== +@match: [[ ${1} =~ \.(lisp|lsp|cl)$ ]] +[[ ${1} =~ \.(lisp|lsp|cl)$ ]] +=== +@match: [[ a == - ]] +[[ a == - ]] +=== +@match: run_test_command() [[ -e foo ]] +run_test_command() [[ -e foo ]] +=== +@known-diff test-extglob-rejected-group: [[ $a == *.@(txt|log) ]] +[[ $a == *.@(txt|log) ]] +--- +hasError: true +program [0,24] "[[ $a == *.@(txt|log) ]]" + test_command [0,24] "[[ $a == *.@(txt|log) ]]" + ([[) [0,2] "[[" + binary_expression [3,12] "$a == *.@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,12] "*.@" + ERROR [12,22] "(txt|log) " + (]]) [22,24] "]]" +=== +@known-diff test-extglob-rejected-group: [[ $a == x@(y) ]] +[[ $a == x@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == x@(y) ]]" + test_command [0,17] "[[ $a == x@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == x@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] "x@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@match: [[ $a == ?(a) ]] +[[ $a == ?(a) ]] +=== +@known-diff test-pattern-two-constructs: [[ $a == *${x}*${y} ]] +[[ $a == *${x}*${y} ]] +--- +hasError: false +program [0,22] "[[ $a == *${x}*${y} ]]" + test_command [0,22] "[[ $a == *${x}*${y} ]]" + ([[) [0,2] "[[" + binary_expression [3,19] "$a == *${x}*${y}" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + extglob_pattern [9,10] "*" + expansion [10,14] "${x}" + (${) [10,12] "${" + variable_name [12,13] "x" + (}) [13,14] "}" + extglob_pattern [14,19] "*${y}" + (]]) [20,22] "]]" +=== +@match: [[ $a == ${x}* ]] +[[ $a == ${x}* ]] +=== +@known-diff test-pattern-two-constructs: [[ $a == *"s"*"t" ]] +[[ $a == *"s"*"t" ]] +--- +hasError: false +program [0,20] "[[ $a == *\"s\"*\"t\" ]]" + test_command [0,20] "[[ $a == *\"s\"*\"t\" ]]" + ([[) [0,2] "[[" + binary_expression [3,17] "$a == *\"s\"*\"t\"" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + extglob_pattern [9,10] "*" + string [10,13] "\"s\"" + (") [10,11] "\"" + string_content [11,12] "s" + (") [12,13] "\"" + extglob_pattern [13,17] "*\"t\"" + (]]) [18,20] "]]" +=== +@match: [ -f x ] +[ -f x ] +=== +@match: [ ! -f x ] +[ ! -f x ] +=== +@match: [ a = b ] 2>/dev/null +[ a = b ] 2>/dev/null +=== +@match: [[ $a == @(y) ]] +[[ $a == @(y) ]] +=== +@known-diff test-extglob-rejected-group: [[ $a == *@(y) ]] +[[ $a == *@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == *@(y) ]]" + test_command [0,17] "[[ $a == *@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == *@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] "*@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@match: [[ $a == */@(y) ]] +[[ $a == */@(y) ]] +=== +@known-diff test-extglob-rejected-group: [[ $a == .@(y) ]] +[[ $a == .@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == .@(y) ]]" + test_command [0,17] "[[ $a == .@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == .@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] ".@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@known-diff test-extglob-rejected-group: [[ $a == _@(y) ]] +[[ $a == _@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == _@(y) ]]" + test_command [0,17] "[[ $a == _@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == _@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] "_@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@match: [[ $a == ?*(y) ]] +[[ $a == ?*(y) ]] +=== +@match: [[ $a == a*(y) ]] +[[ $a == a*(y) ]] +=== +@match: [[ $a == *?(y) ]] +[[ $a == *?(y) ]] +=== +@match: [[ $a =~ 'a'b ]] +[[ $a =~ 'a'b ]] +=== +@match: [[ $a =~ a'b'c ]] +[[ $a =~ a'b'c ]] +=== +@match: [ ! command -v go ] +[ ! command -v go ] +=== +@match: [ command -v go &>/dev/null ] +[ command -v go &>/dev/null ] +=== diff --git a/packages/tree-sitter-bash/test/fuzz.test.ts b/packages/tree-sitter-bash/test/fuzz.test.ts new file mode 100644 index 0000000000..cecbbbcc12 --- /dev/null +++ b/packages/tree-sitter-bash/test/fuzz.test.ts @@ -0,0 +1,165 @@ +// test/fuzz.test.ts +// +// Deterministic fuzzing (fixed-seed PRNG — no per-run randomness; scale the +// volume with TS_BASH_FUZZ_SCALE=N for deeper local runs): +// a) token soup — random sequences of bash operators/keywords/fragments; +// b) byte mutations of the curated fixtures (replace/delete/insert, +// NUL included); +// c) nesting bombs — programmatic deep $(…) / (…) / ${…} / if nesting. +// +// Contract under test: parse never throws; within budget it returns either +// ok:true or { ok:false, reason:'aborted' }; an ok:true tree is always +// structurally sound (assertTreeIntegrity); nesting beyond the documented +// depth caps degrades locally (hasError) instead of throwing. + +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { parse } from '#/parse'; +import type { ParseOptions, ParseResult } from '#/parse'; +import { assertTreeIntegrity, parseFixtureFile } from './helpers/differential'; + +const PKG_ROOT = path.resolve(import.meta.dirname, '..'); +const SCALE = Math.max(1, Number(process.env['TS_BASH_FUZZ_SCALE'] ?? 1) || 1); + +/** Park–Miller minimal standard PRNG: small, portable, deterministic. */ +function prng(seed: number): () => number { + let state = seed % 2147483647; + if (state <= 0) state += 2147483646; + return () => { + state = (state * 16807) % 2147483647; + return (state - 1) / 2147483646; + }; +} + +function pick(rand: () => number, pool: readonly string[]): string { + return pool[Math.floor(rand() * pool.length)]!; +} + +function checkContract(source: string, options?: ParseOptions): ParseResult { + let result: ParseResult | undefined; + expect(() => { + result = parse(source, options); + }).not.toThrow(); + expect(result).toBeDefined(); + if (result!.ok) { + assertTreeIntegrity(result!.rootNode, source); + } else { + expect(result!).toEqual({ ok: false, reason: 'aborted' }); + } + return result!; +} + +const TOKEN_POOL = [ + 'echo', 'ls', 'foo', 'bar', 'x', 'A=1', 'if', 'then', 'fi', 'while', 'do', 'done', 'for', 'in', 'case', 'esac', + '&&', '||', '|', '|&', ';', '&', ';;', ';&', '(', ')', '{', '}', '>', '>>', '<', '>&1', '2>', '&>', '<(p)', '<(p)', '$#', '$?', '0x1F', '..', '<<-', '>&-', 'abc_def', +] as const; + +function fixtureSources(): string[] { + const dir = path.join(PKG_ROOT, 'test/fixtures/differential'); + const out: string[] = []; + for (const file of readdirSync(dir).filter((f) => f.endsWith('.txt')).toSorted()) { + for (const sample of parseFixtureFile(file, readFileSync(path.join(dir, file), 'utf8'))) { + out.push(sample.source); + } + } + return out; +} + +describe('fuzz: token soup', () => { + it('never throws and always yields a sound tree or a clean abort', () => { + const rand = prng(0x5eed0001); + const count = 250 * SCALE; + let parsed = 0; + for (let n = 0; n < count; n++) { + const length = 3 + Math.floor(rand() * 22); + const parts: string[] = []; + for (let k = 0; k < length; k++) parts.push(pick(rand, TOKEN_POOL)); + if (checkContract(parts.join(' ')).ok) parsed++; + } + expect(parsed).toBeGreaterThan(0); + console.log(`token soup: ${count} inputs, ${parsed} parsed / ${count - parsed} aborted`); + }); +}); + +describe('fuzz: byte mutations of fixtures', () => { + it('never throws and always yields a sound tree or a clean abort', () => { + const rand = prng(0x5eed0002); + const bases = fixtureSources(); + const count = 300 * SCALE; + let parsed = 0; + for (let n = 0; n < count; n++) { + const base = pick(rand, bases); + if (base.length === 0) continue; + const pos = Math.floor(rand() * base.length); + const mode = rand(); + let mutated: string; + if (mode < 0.4) { + // replace one code unit (may be NUL) + const code = Math.floor(rand() * 256); + mutated = base.slice(0, pos) + String.fromCodePoint(code) + base.slice(pos + 1); + } else if (mode < 0.7) { + mutated = base.slice(0, pos) + base.slice(pos + 1); + } else { + const code = Math.floor(rand() * 256); + mutated = base.slice(0, pos) + String.fromCodePoint(code) + base.slice(pos); + } + if (checkContract(mutated).ok) parsed++; + } + expect(parsed).toBeGreaterThan(0); + console.log(`byte mutations: ${count} inputs, ${parsed} parsed / ${count - parsed} aborted`); + }); +}); + +describe('fuzz: nesting bombs degrade locally per the documented depth caps', () => { + const substitution = (depth: number): string => `echo ${'$('.repeat(depth)}x${')'.repeat(depth)}`; + const subshell = (depth: number): string => `${'('.repeat(depth)}x${')'.repeat(depth)}`; + const expansion = (depth: number): string => `echo ${'${a:-'.repeat(depth)}z${'}'.repeat(depth)}`; + const ifs = (depth: number): string => `${'if x; then '.repeat(depth)}y${'; fi'.repeat(depth)}`; + + it('within the caps the trees are clean', () => { + // literalDepth ticks twice per ${…} level (parseLiteral + parseExpansion), + // so 200 levels stay under MAX_PARSE_DEPTH = 500. + for (const source of [substitution(100), subshell(400), expansion(200), ifs(400)]) { + const result = checkContract(source, { timeoutMs: 60_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(false); + } + }); + + it('beyond the caps the parse still succeeds and flags hasError (MAX_SUBSTITUTION_DEPTH = 150)', () => { + const result = checkContract(substitution(200)); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('beyond the caps the parse still succeeds and flags hasError (MAX_PARSE_DEPTH = 500)', () => { + for (const source of [subshell(600), expansion(600), ifs(600)]) { + const result = checkContract(source, { timeoutMs: 60_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('extreme nesting never overflows the stack', () => { + for (const source of [substitution(5000), subshell(5000), expansion(5000), ifs(5000)]) { + const result = checkContract(source, { timeoutMs: 60_000 }); + // Either a locally degraded tree (hasError) or a clean budget abort. + if (result.ok) expect(result.hasError).toBe(true); + else expect(result.reason).toBe('aborted'); + } + }); + + it('the node budget aborts huge flat programs, and a raised budget parses them', () => { + const source = 'echo a; '.repeat(20_000); + expect(parse(source)).toEqual({ ok: false, reason: 'aborted' }); + const result = parse(source, { timeoutMs: 60_000, maxNodes: 10_000_000 }); + expect(result.ok).toBe(true); + if (result.ok) assertTreeIntegrity(result.rootNode, source); + }); +}); diff --git a/packages/tree-sitter-bash/test/helpers/differential.ts b/packages/tree-sitter-bash/test/helpers/differential.ts new file mode 100644 index 0000000000..e66e5ebe82 --- /dev/null +++ b/packages/tree-sitter-bash/test/helpers/differential.ts @@ -0,0 +1,271 @@ +// test/helpers/differential.ts +// +// Differential test harness: parses the same source with both this package's +// parser and the real tree-sitter-bash (web-tree-sitter + the official wasm +// build shipped in the tree-sitter-bash npm package), normalizes both trees +// into a comparable dump, and produces structured diffs. +// +// Normalization: a pre-order dump of EVERY node (named and anonymous — the +// byte-identical trees M1/M2 verified include the anonymous token layer), +// one line per node: `type [start,end] "text preview"`, anonymous +// types wrapped in parens, plus a leading `hasError:` line. Both sides use +// UTF-16 code unit offsets directly: web-tree-sitter (wasm) reports UTF-16 +// offsets for string input, same as this parser (verified: for +// "echo 你好 && ls" the reference reports the root as [0,13] and `&&` as +// [8,10] — UTF-16 code units, not UTF-8 bytes). + +import path from 'node:path'; + +import { Language, Parser as RefParser } from 'web-tree-sitter'; +import type { Node as RefNode } from 'web-tree-sitter'; + +import type { SyntaxNode } from '#/node'; +import { parse } from '#/parse'; + +const PACKAGE_ROOT = path.resolve(import.meta.dirname, '../..'); +const WASM_PATH = path.join(PACKAGE_ROOT, 'node_modules/tree-sitter-bash/tree-sitter-bash.wasm'); + +let refParserPromise: Promise | null = null; + +/** + * Lazily load web-tree-sitter + the bash wasm exactly once per test process. + * Load failures throw a descriptive error (never silently skip): the + * differential suite is meaningless without the reference. + */ +export function loadReferenceParser(): Promise { + refParserPromise ??= (async () => { + try { + await RefParser.init(); + const language = await Language.load(WASM_PATH); + const parser = new RefParser(); + parser.setLanguage(language); + return parser; + } catch (error) { + throw new Error( + `failed to load the tree-sitter-bash wasm reference (expected at ${WASM_PATH}). ` + + 'Run `pnpm install` and make sure the tree-sitter-bash devDependency is present.', + { cause: error }, + ); + } + })(); + return refParserPromise; +} + +interface DumpLine { + depth: number; + label: string; + start: number; + end: number; + text: string; +} + +function render(lines: DumpLine[], hasError: boolean): string { + const out = [`hasError: ${hasError}`]; + for (const line of lines) { + const preview = + line.text.length <= 40 ? JSON.stringify(line.text) : JSON.stringify(`${line.text.slice(0, 37)}...`); + out.push(`${' '.repeat(line.depth)}${line.label} [${line.start},${line.end}] ${preview}`); + } + return out.join('\n'); +} + +function label(type: string, isNamed: boolean): string { + return isNamed ? type : `(${type})`; +} + +/** Normalized dump of the reference tree (UTF-16 offsets as reported). */ +export async function referenceDump(source: string): Promise { + const parser = await loadReferenceParser(); + const tree = parser.parse(source); + if (tree === null) throw new Error(`reference parser returned null for ${JSON.stringify(source)}`); + const lines: DumpLine[] = []; + const stack: Array<{ node: RefNode; depth: number }> = [{ node: tree.rootNode, depth: 0 }]; + while (stack.length > 0) { + const { node, depth } = stack.pop()!; + lines.push({ + depth, + label: label(node.type, node.isNamed), + start: node.startIndex, + end: node.endIndex, + text: source.slice(node.startIndex, node.endIndex), + }); + for (let i = node.childCount - 1; i >= 0; i--) stack.push({ node: node.child(i)!, depth: depth + 1 }); + } + return render(lines, tree.rootNode.hasError); +} + +/** + * Normalized dump of our tree. Parses with a generous budget so differential + * results are never polluted by the default 50 ms / 50 000-node budget; + * fixtures are expected to parse, so an abort here is an error. + */ +export function ourDump(source: string): string { + const result = parse(source, { timeoutMs: 60_000, maxNodes: 10_000_000 }); + if (!result.ok) throw new Error(`our parser aborted on a differential fixture: ${JSON.stringify(source)}`); + const lines: DumpLine[] = []; + const stack: Array<{ node: SyntaxNode; depth: number }> = [{ node: result.rootNode, depth: 0 }]; + while (stack.length > 0) { + const { node, depth } = stack.pop()!; + lines.push({ depth, label: label(node.type, node.isNamed), start: node.startIndex, end: node.endIndex, text: node.text }); + for (let i = node.children.length - 1; i >= 0; i--) stack.push({ node: node.children[i]!, depth: depth + 1 }); + } + return render(lines, result.hasError); +} + +/** First-difference report between two dumps; empty string when identical. */ +export function diffDumps(ours: string, reference: string): string { + if (ours === reference) return ''; + const a = ours.split('\n'); + const b = reference.split('\n'); + let i = 0; + while (i < a.length && i < b.length && a[i] === b[i]) i++; + const slice = (lines: string[], from: number): string => + lines + .slice(from, from + 8) + .map((line, j) => `${from + j === i ? '>' : ' '} ${line}`) + .join('\n'); + const at = Math.max(0, i - 2); + return ( + `trees differ at dump line ${i + 1}:\n` + + ` ours:\n${slice(a, at)}\n` + + ` reference:\n${slice(b, at)}` + ); +} + +export interface Comparison { + equal: boolean; + ours: string; + reference: string; + diff: string; +} + +/** Parse `source` with both parsers and compare the normalized dumps. */ +export async function compareSource(source: string): Promise { + const [ours, reference] = [ourDump(source), await referenceDump(source)]; + return { equal: ours === reference, ours, reference, diff: diffDumps(ours, reference) }; +} + +/** + * Tree integrity self-check (used by fuzz): every node range must lie inside + * the source, `text` must equal the corresponding slice, and children must be + * contained in their parent, in source order, without overlaps. Throws with a + * descriptive message on the first violation. + */ +export function assertTreeIntegrity(root: SyntaxNode, source: string): void { + const stack: SyntaxNode[] = [root]; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.startIndex < 0 || node.endIndex < node.startIndex || node.endIndex > source.length) { + throw new Error(`node ${node.type} range [${node.startIndex}, ${node.endIndex}) escapes source length ${source.length}`); + } + if (node.text !== source.slice(node.startIndex, node.endIndex)) { + throw new Error(`node ${node.type} text does not match source.slice(${node.startIndex}, ${node.endIndex})`); + } + let previousEnd = node.startIndex; + for (const child of node.children) { + if (child.startIndex < node.startIndex || child.endIndex > node.endIndex) { + throw new Error(`child ${child.type} escapes parent ${node.type}`); + } + if (child.startIndex < previousEnd) { + throw new Error(`child ${child.type} overlaps its previous sibling in ${node.type}`); + } + previousEnd = child.endIndex; + stack.push(child); + } + } +} + +// ------------------------------------------------------------- fixture I/O + +/** A curated differential fixture sample (see test/fixtures/differential/). */ +export interface FixtureSample { + /** Directive: 'match' (must equal the reference) or 'known-diff'. */ + kind: 'match' | 'known-diff'; + /** Known-difference registry id (known-diff samples only). */ + id?: string; + /** Free-form description from the directive line. */ + description: string; + source: string; + /** Stored expected dump of OUR parser (known-diff samples only). */ + expectedOurs?: string; + /** 1-based line of the directive, for error messages. */ + line: number; +} + +const DIRECTIVE_RE = /^@(match|known-diff)(?:\s+(\S+))?:\s*(.*)$/; +const SEPARATOR_RE = /^===\s*$/m; +const DUMP_SPLIT_RE = /^---\s*$/m; + +/** One official tree-sitter-bash corpus test case (input only — the + * expected S-expression is ignored; the live wasm reference is the + * comparison target). */ +export interface CorpusCase { + name: string; + input: string; +} + +/** Parse an official corpus file (`==== name ====` / input / `----` / + * expected). The separator is any run of 3+ dashes. */ +export function parseCorpusFile(content: string): CorpusCase[] { + const out: CorpusCase[] = []; + const lines = content.split('\n'); + let i = 0; + while (i < lines.length) { + if (!/^=+\s*$/.test(lines[i]!)) { + i++; + continue; + } + i++; + const nameLines: string[] = []; + while (i < lines.length && !/^=+\s*$/.test(lines[i]!)) nameLines.push(lines[i++]!); + i++; + const inputLines: string[] = []; + while (i < lines.length && !/^-{3,}\s*$/.test(lines[i]!)) inputLines.push(lines[i++]!); + i++; + while (i < lines.length && !/^=+\s*$/.test(lines[i]!)) i++; + out.push({ name: nameLines.join(' ').trim(), input: inputLines.join('\n').replace(/^\n+/, '').replace(/\n+$/, '') }); + } + return out; +} + + +/** + * Parse a fixture file. Format: blocks separated by lines of `===`; each + * block starts with `@match: ` or `@known-diff : `, + * followed by the sample source. known-diff blocks additionally carry the + * expected dump of our parser after a `---` line (the disclosed deviation + * shape — if our output drifts from it, or starts matching the reference, + * the test fails). + */ +export function parseFixtureFile(filePath: string, content: string): FixtureSample[] { + const samples: FixtureSample[] = []; + const blocks = content.split(SEPARATOR_RE); + let line = 1; + for (const rawBlock of blocks) { + const block = rawBlock.replace(/^\n/, '').replace(/\n$/, ''); + const blockLine = line; + line += rawBlock.split('\n').length; + if (block.trim().length === 0) continue; + const nl = block.indexOf('\n'); + const directiveLine = nl === -1 ? block : block.slice(0, nl); + const body = nl === -1 ? '' : block.slice(nl + 1); + const directive = DIRECTIVE_RE.exec(directiveLine.trim()); + if (directive === null) { + throw new Error(`${filePath}:${blockLine}: expected a @match: / @known-diff : directive, got ${JSON.stringify(directiveLine)}`); + } + const [, kind, id, description = ''] = directive; + if (kind === 'match') { + samples.push({ kind, description, source: body, line: blockLine }); + } else { + if (id === undefined) throw new Error(`${filePath}:${blockLine}: @known-diff requires a registry id`); + const parts = body.split(DUMP_SPLIT_RE); + if (parts.length !== 2) { + throw new Error(`${filePath}:${blockLine}: @known-diff block must contain a --- line followed by the expected dump of our parser`); + } + const source = parts[0]!.replace(/\n$/, ''); + const expectedOurs = parts[1]!.replace(/^\n/, '').replace(/\n$/, ''); + samples.push({ kind: 'known-diff', id, description, source, expectedOurs, line: blockLine }); + } + } + return samples; +} diff --git a/packages/tree-sitter-bash/test/helpers/known-differences.ts b/packages/tree-sitter-bash/test/helpers/known-differences.ts new file mode 100644 index 0000000000..b3b526e6f6 --- /dev/null +++ b/packages/tree-sitter-bash/test/helpers/known-differences.ts @@ -0,0 +1,179 @@ +// test/helpers/known-differences.ts +// +// The single source of truth for the known, deliberately documented ways +// this parser's tree deviates from tree-sitter-bash 0.25.0. Every entry is: +// - referenced by @known-diff samples in test/fixtures/differential/ and +// test/fixtures/corpus/known-diffs.txt (the stored dump in each sample +// pins the exact deviation shape — if our output drifts, or starts +// matching the reference, the differential tests fail); +// - anchored in README.md's "Known differences" section: `readmeAnchor` +// must appear there verbatim, so the list and the README cannot +// silently drift apart (enforced by test/differential.test.ts). + +export interface KnownDifference { + /** Identifier referenced from fixture @known-diff directives. */ + readonly id: string; + /** Verbatim substring of README.md's Known differences section. */ + readonly readmeAnchor: string; + /** Short human-readable description. */ + readonly summary: string; +} + +export const KNOWN_DIFFERENCES: readonly KnownDifference[] = [ + { + id: 'redirect-readwrite', + readmeAnchor: '`<>` (read-write redirect) is parsed as a normal `file_redirect`', + summary: '<> parses as file_redirect; the reference fails to parse it.', + }, + { + id: 'heredoc-multiple-on-line', + readmeAnchor: 'Several heredocs on one line', + summary: 'cat < is a word in the reference (scanner-state quirk); a continuation INSIDE a pattern is an error there. This parser classifies by content and keeps continuations.', + }, + { + id: 'nonascii-identifier-assignment', + readmeAnchor: 'A non-ASCII “identifier” in assignment position', + summary: '变量=值 is a hasError variable_assignment in the reference; this parser keeps it a plain command word (bash itself rejects such names).', + }, + { + id: 'case-rejected-group', + readmeAnchor: 'An extglob group the reference rejects in a case pattern', + summary: 'x@(y)): the reference reparses the group as a new case item; this parser degrades the whole pattern to ERROR.', + }, + { + id: 'expansion-replacement-escaped-backslash', + readmeAnchor: 'A double backslash in a replacement value (`${x// /\\\\|}`)', + summary: 'The reference drops the first backslash (a quirk); this parser keeps both.', + }, +]; + +const KNOWN_DIFFERENCE_IDS: ReadonlySet = new Set(KNOWN_DIFFERENCES.map((entry) => entry.id)); + +export function isKnownDifferenceId(id: string): boolean { + return KNOWN_DIFFERENCE_IDS.has(id); +} diff --git a/packages/tree-sitter-bash/test/node.test.ts b/packages/tree-sitter-bash/test/node.test.ts new file mode 100644 index 0000000000..16d5219808 --- /dev/null +++ b/packages/tree-sitter-bash/test/node.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest'; + +import { createNode, descendantsOfType } from '#/node'; + +const SOURCE = 'echo foo; echo bar'; + +/** program + * ├─ command "echo foo" + * │ ├─ command_name "echo" + * │ │ └─ word "echo" + * │ └─ word "foo" + * ├─ ";" (anonymous) + * └─ command "echo bar" + * └─ word "bar" + */ +function buildTree() { + const program = createNode({ type: 'program', source: SOURCE, startIndex: 0, endIndex: SOURCE.length }); + const cmd1 = program.addChild(createNode({ type: 'command', source: SOURCE, startIndex: 0, endIndex: 8 })); + const name1 = cmd1.addChild(createNode({ type: 'command_name', source: SOURCE, startIndex: 0, endIndex: 4 })); + name1.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 })); + cmd1.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 5, endIndex: 8 })); + program.addChild(createNode({ type: ';', source: SOURCE, startIndex: 8, endIndex: 9, isNamed: false })); + const cmd2 = program.addChild(createNode({ type: 'command', source: SOURCE, startIndex: 10, endIndex: 18 })); + cmd2.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 15, endIndex: 18 })); + return program; +} + +describe('createNode', () => { + it('pre-stores text as the UTF-16 slice of the source', () => { + const node = createNode({ type: 'word', source: SOURCE, startIndex: 5, endIndex: 8 }); + expect(node.text).toBe('foo'); + expect(node.startIndex).toBe(5); + expect(node.endIndex).toBe(8); + expect(node.isNamed).toBe(true); + expect(node.parent).toBeNull(); + }); + + it('rejects out-of-range offsets', () => { + expect(() => createNode({ type: 'word', source: SOURCE, startIndex: -1, endIndex: 2 })).toThrow(RangeError); + expect(() => createNode({ type: 'word', source: SOURCE, startIndex: 3, endIndex: 2 })).toThrow(RangeError); + expect(() => createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 100 })).toThrow(RangeError); + }); + + it('rejects attaching a child that already has a parent', () => { + const a = createNode({ type: 'program', source: SOURCE, startIndex: 0, endIndex: SOURCE.length }); + const b = createNode({ type: 'program', source: SOURCE, startIndex: 0, endIndex: SOURCE.length }); + const child = createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 }); + a.addChild(child); + expect(() => b.addChild(child)).toThrow(/already has a parent/); + }); + + it('rejects children outside the parent range', () => { + const parent = createNode({ type: 'command', source: SOURCE, startIndex: 0, endIndex: 8 }); + expect(() => + parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 9 })), + ).toThrow(RangeError); + }); + + it('rejects overlapping or out-of-order siblings', () => { + const parent = createNode({ type: 'command', source: SOURCE, startIndex: 0, endIndex: 8 }); + parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 2, endIndex: 5 })); + expect(() => parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 4, endIndex: 7 }))).toThrow( + RangeError, + ); + expect(() => parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 2 }))).toThrow( + RangeError, + ); + // Adjacent (start == previous end) is fine. + parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 5, endIndex: 8 })); + expect(parent.children).toHaveLength(2); + }); +}); + +describe('children vs namedChildren', () => { + it('keeps anonymous nodes out of namedChildren but in children', () => { + const program = buildTree(); + expect(program.children.map((c) => c.type)).toEqual(['command', ';', 'command']); + expect(program.namedChildren.map((c) => c.type)).toEqual(['command', 'command']); + expect(program.children[1]?.isNamed).toBe(false); + }); + + it('wires parent pointers', () => { + const program = buildTree(); + const [cmd1] = program.namedChildren; + expect(cmd1?.parent).toBe(program); + expect(cmd1?.namedChildren[0]?.parent).toBe(cmd1); + }); +}); + +describe('descendantsOfType', () => { + it('returns matching named descendants in pre-order', () => { + const program = buildTree(); + const words = descendantsOfType(program, 'word'); + expect(words.map((w) => w.text)).toEqual(['echo', 'foo', 'bar']); + }); + + it('matches multiple types and skips anonymous nodes', () => { + const program = buildTree(); + const nodes = descendantsOfType(program, 'command', ';'); + expect(nodes.map((n) => n.type)).toEqual(['command', 'command']); + }); + + it('returns every named descendant when no type is given', () => { + const program = buildTree(); + expect(descendantsOfType(program).map((n) => n.type)).toEqual([ + 'command', + 'command_name', + 'word', + 'word', + 'command', + 'word', + ]); + }); + + it('does not include the root itself', () => { + const leaf = createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 }); + expect(descendantsOfType(leaf, 'word')).toEqual([]); + }); + + it('handles pathologically deep trees without overflowing the stack', () => { + // 200k-deep chain; a recursive walk would overflow the call stack. + const depth = 200_000; + const nodes = []; + for (let i = 0; i < depth; i++) { + nodes.push(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 1 })); + } + for (let i = depth - 2; i >= 0; i--) { + nodes[i]!.addChild(nodes[i + 1]!); + } + expect(descendantsOfType(nodes[0]!, 'word')).toHaveLength(depth - 1); + }); +}); diff --git a/packages/tree-sitter-bash/test/parse.test.ts b/packages/tree-sitter-bash/test/parse.test.ts new file mode 100644 index 0000000000..bb322741f4 --- /dev/null +++ b/packages/tree-sitter-bash/test/parse.test.ts @@ -0,0 +1,519 @@ +import { describe, expect, it } from 'vitest'; + +import { Aborted, ParseBudget } from '#/budget'; +import type { SyntaxNode } from '#/node'; +import { descendantsOfType } from '#/node'; +import { parse } from '#/parse'; + +/** Compact S-expression of the tree: named leaves are (type "text"), + * anonymous leaves are just their quoted type, inner nodes nest. */ +function sexp(node: SyntaxNode): string { + if (node.children.length === 0) { + return node.isNamed ? `(${node.type} ${JSON.stringify(node.text)})` : JSON.stringify(node.type); + } + const label = node.isNamed ? node.type : JSON.stringify(node.type); + return `(${label} ${node.children.map(sexp).join(' ')})`; +} + +function parseOk(source: string): { rootNode: SyntaxNode; hasError: boolean } { + const result = parse(source); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error('unreachable'); + return result; +} + +/** Parse and assert the exact tree shape plus the hasError flag. */ +function expectTree(source: string, expected: string, hasError = false): SyntaxNode { + const { rootNode, hasError: actual } = parseOk(source); + expect(actual).toBe(hasError); + expect(sexp(rootNode)).toBe(expected); + return rootNode; +} + +describe('ParseBudget', () => { + it('counts nodes and stays under the cap', () => { + const budget = new ParseBudget({ timeoutMs: 60_000, maxNodes: 3 }); + budget.tick(); + budget.tick(); + expect(budget.nodesUsed).toBe(2); + }); + + it('throws Aborted when the node cap is exceeded', () => { + const budget = new ParseBudget({ timeoutMs: 60_000, maxNodes: 1 }); + budget.tick(); + expect(() => { + budget.tick(); + }).toThrow(Aborted); + }); + + it('throws Aborted once the deadline is reached', () => { + const budget = new ParseBudget({ timeoutMs: 0, maxNodes: 1_000 }); + expect(() => { + budget.tick(); + }).toThrow(Aborted); + }); + + it('applies documented defaults', () => { + const budget = new ParseBudget(); + // Default node cap is 50_000: the 50_000th node is still allowed. + for (let i = 0; i < 50_000; i++) budget.tick(); + expect(() => { + budget.tick(); + }).toThrow(Aborted); + }); +}); + +describe('parse entry contract', () => { + it('parses an empty source into an empty program', () => { + const { rootNode, hasError } = parseOk(''); + expect(rootNode.type).toBe('program'); + expect(rootNode.text).toBe(''); + expect(rootNode.children).toHaveLength(0); + expect(hasError).toBe(false); + }); + + it('returns { ok: false, reason: "aborted" } when the node budget is exceeded', () => { + expect(parse('echo hello', { timeoutMs: 60_000, maxNodes: 1 })).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('returns { ok: false, reason: "aborted" } when the deadline has passed', () => { + expect(parse('echo hello', { timeoutMs: 0 })).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('aborts instead of hanging on tens of thousands of statements', () => { + expect(parse('ls; '.repeat(20_000))).toEqual({ ok: false, reason: 'aborted' }); + }); +}); + +describe('simple commands', () => { + it('parses a command with arguments', () => { + expectTree('ls -la', `(program (command (command_name (word "ls")) (word "-la")))`); + }); + + it('parses double-quoted strings with expansions and raw strings', () => { + expectTree( + `echo "hello $NAME" 'raw'`, + `(program (command (command_name (word "echo")) (string "\\"" (string_content "hello ") (simple_expansion "$" (variable_name "NAME")) "\\"") (raw_string "'raw'")))`, + ); + }); + + it('parses a pipeline inside && / || lists, left associative', () => { + expectTree( + 'FOO=bar env | grep FOO && echo ok || echo fail', + `(program (list (list (pipeline (command (variable_assignment (variable_name "FOO") "=" (word "bar")) (command_name (word "env"))) "|" (command (command_name (word "grep")) (word "FOO"))) "&&" (command (command_name (word "echo")) (word "ok"))) "||" (command (command_name (word "echo")) (word "fail"))))`, + ); + }); + + it('parses ! as negated_command', () => { + expectTree( + '! grep -q x file', + `(program (negated_command "!" (command (command_name (word "grep")) (word "-q") (word "x") (word "file"))))`, + ); + }); + + it('attaches comments as named children', () => { + expectTree( + '# a comment\necho hi # trailing', + `(program (comment "# a comment") (command (command_name (word "echo")) (word "hi")) (comment "# trailing"))`, + ); + }); + + it('distinguishes number from word', () => { + expectTree( + 'echo 123 45.6', + `(program (command (command_name (word "echo")) (number "123") (word "45.6")))`, + ); + }); + + it('parses concatenations of adjacent pieces', () => { + expectTree( + `echo a"b"c'd'$e\${f}g`, + `(program (command (command_name (word "echo")) (concatenation (word "a") (string "\\"" (string_content "b") "\\"") (word "c") (raw_string "'d'") (simple_expansion "$" (variable_name "e")) (expansion "\${" (variable_name "f") "}") (word "g"))))`, + ); + }); + + it('splits { } into single-character word pieces', () => { + expectTree( + 'echo {a,b}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "a,b") (word "}"))))`, + ); + }); + + it('treats a lone $ as an anonymous argument', () => { + expectTree('echo $', `(program (command (command_name (word "echo")) "$"))`); + }); +}); + +describe('expansions and substitutions', () => { + it('parses backtick and $() command substitutions', () => { + expectTree( + 'echo `hostname` $(date)', + `(program (command (command_name (word "echo")) (command_substitution "\`" (command (command_name (word "hostname"))) "\`") (command_substitution "$(" (command (command_name (word "date"))) ")")))`, + ); + }); + + it('parses nested command substitutions recursively', () => { + expectTree( + 'echo $(a $(b c))', + `(program (command (command_name (word "echo")) (command_substitution "$(" (command (command_name (word "a")) (command_substitution "$(" (command (command_name (word "b")) (word "c")) ")")) ")")))`, + ); + }); + + it('parses ${...} expansions with operators and defaults', () => { + expectTree( + 'echo ${v:-d} ${x} $1 $@', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (word "d") "}") (expansion "\${" (variable_name "x") "}") (simple_expansion "$" (variable_name "1")) (simple_expansion "$" (special_variable_name "@"))))`, + ); + }); + + it('parses ${#v}, ${v:=word} and ${a[0]}', () => { + expectTree('echo ${#v}', `(program (command (command_name (word "echo")) (expansion "\${" "#" (variable_name "v") "}")))`); + expectTree( + 'echo ${v:=word}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":=" (word "word") "}")))`, + ); + expectTree( + 'echo ${a[0]}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "a") "[" (number "0") "]") "}")))`, + ); + }); + + it('parses expansions nested inside expansions', () => { + expectTree( + 'echo ${v:-$(cmd)}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (command_substitution "$(" (command (command_name (word "cmd"))) ")") "}")))`, + ); + }); + + it('parses process substitutions', () => { + expectTree( + 'diff <(cmd) >(other)', + `(program (command (command_name (word "diff")) (process_substitution "<(" (command (command_name (word "cmd"))) ")") (process_substitution ">(" (command (command_name (word "other"))) ")")))`, + ); + }); + + it('parses $((...)) as a real arithmetic expression', () => { + expectTree( + 'echo $((1<<2))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (number "1") "<<" (number "2")) "))")))`, + ); + }); +}); + +describe('redirects', () => { + it('parses output redirect and fd duplication', () => { + expectTree( + 'cmd > out 2>&1', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect ">" (word "out")) (file_redirect (file_descriptor "2") ">&" (number "1"))))`, + ); + }); + + it('parses &>> appending both streams', () => { + expectTree( + 'cmd &>> log', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect "&>>" (word "log"))))`, + ); + }); + + it('parses <> read-write redirect with a file descriptor', () => { + expectTree( + 'exec 3<>file', + `(program (redirected_statement (command (command_name (word "exec"))) (file_redirect (file_descriptor "3") "<>" (word "file"))))`, + ); + }); + + it('parses >|, <&-, < and fd redirects', () => { + expectTree( + 'cmd >| file', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect ">|" (word "file"))))`, + ); + expectTree('cmd >&-', `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect ">&-")))`); + expectTree( + 'cmd < in', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect "<" (word "in"))))`, + ); + expectTree( + 'cmd 2>/dev/null', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect (file_descriptor "2") ">" (word "/dev/null"))))`, + ); + }); + + it('keeps a herestring inside the command node', () => { + expectTree( + 'cmd <<< "$str"', + `(program (command (command_name (word "cmd")) (herestring_redirect "<<<" (string "\\"" (simple_expansion "$" (variable_name "str")) "\\""))))`, + ); + }); + + it('wraps a command carrying both a herestring and a file redirect', () => { + expectTree( + 'cmd <<< x > out', + `(program (redirected_statement (command (command_name (word "cmd")) (herestring_redirect "<<<" (word "x"))) (file_redirect ">" (word "out"))))`, + ); + }); + + it('parses a redirect-only statement', () => { + expectTree('> out', `(program (redirected_statement (file_redirect ">" (word "out"))))`); + }); + + it('keeps a prefix redirect inside the command', () => { + expectTree( + '> a cmd x', + `(program (command (file_redirect ">" (word "a")) (command_name (word "cmd")) (word "x")))`, + ); + }); + + it('wraps subshells and negated commands with their redirects', () => { + expectTree( + '(ls) > out', + `(program (redirected_statement (subshell "(" (command (command_name (word "ls"))) ")") (file_redirect ">" (word "out"))))`, + ); + expectTree( + '! ls > out', + `(program (redirected_statement (negated_command "!" (command (command_name (word "ls")))) (file_redirect ">" (word "out"))))`, + ); + }); +}); + +describe('heredocs', () => { + it('parses a heredoc with expansions in the body', () => { + expectTree( + 'cat < { + expectTree( + 'cat <<-EOF\n\tindented\n\tEOF', + `(program (redirected_statement (command (command_name (word "cat"))) (heredoc_redirect "<<-" (heredoc_start "EOF") (heredoc_body (heredoc_content "indented\\n\\t")) (heredoc_end "EOF"))))`, + ); + }); + + it('does not expand the body of a quoted delimiter', () => { + expectTree( + `cat <<'EOF'\nraw $notexpanded\nEOF`, + `(program (redirected_statement (command (command_name (word "cat"))) (heredoc_redirect "<<" (heredoc_start "'EOF'") (heredoc_body "raw $notexpanded\\n") (heredoc_end "EOF"))))`, + ); + }); + + it('absorbs trailing words and redirects into the heredoc_redirect', () => { + expectTree( + 'cat < out\nbody\nEOF', + `(program (redirected_statement (command (command_name (word "cat"))) (heredoc_redirect "<<" (heredoc_start "EOF") (word "arg") (file_redirect ">" (word "out")) (heredoc_body (heredoc_content "body\\n")) (heredoc_end "EOF"))))`, + ); + }); + + it('absorbs a pipeline tail into the heredoc_redirect', () => { + expectTree( + 'cat < { + expectTree( + 'cat < { + expectTree( + 'cat < { + expectTree( + 'cat < { + it('mixes semicolons and newlines as terminators', () => { + expectTree( + 'ls; echo a\necho b', + `(program (command (command_name (word "ls"))) ";" (command (command_name (word "echo")) (word "a")) (command (command_name (word "echo")) (word "b")))`, + ); + }); + + it('parses & background terminators', () => { + expectTree('a & b &', `(program (command (command_name (word "a"))) "&" (command (command_name (word "b"))) "&")`); + }); + + it('parses |& pipelines', () => { + expectTree( + 'cmd1 |& cmd2', + `(program (pipeline (command (command_name (word "cmd1"))) "|&" (command (command_name (word "cmd2")))))`, + ); + }); + + it('parses a subshell', () => { + expectTree('(ls)', `(program (subshell "(" (command (command_name (word "ls"))) ")"))`); + }); + + it('parses standalone assignments', () => { + expectTree('FOO=bar ls', `(program (command (variable_assignment (variable_name "FOO") "=" (word "bar")) (command_name (word "ls"))))`); + expectTree('VAR=value', `(program (variable_assignment (variable_name "VAR") "=" (word "value")))`); + expectTree( + 'A=1 B=2', + `(program (variable_assignments (variable_assignment (variable_name "A") "=" (number "1")) (variable_assignment (variable_name "B") "=" (number "2"))))`, + ); + }); + + it('recovers a trailing && with a partial list and hasError', () => { + expectTree('ls &&', `(program (list (command (command_name (word "ls"))) "&&"))`, true); + }); + + it('recovers a trailing | with a partial pipeline and hasError', () => { + expectTree('ls |', `(program (pipeline (command (command_name (word "ls"))) "|"))`, true); + }); + + it('continues a list across a newline after &&', () => { + expectTree( + 'a &&\nb', + `(program (list (command (command_name (word "a"))) "&&" (command (command_name (word "b")))))`, + ); + }); + + it('splits compound commands into separate command nodes', () => { + const root = expectTree( + 'git status && rm -rf /', + `(program (list (command (command_name (word "git")) (word "status")) "&&" (command (command_name (word "rm")) (word "-rf") (word "/"))))`, + ); + const commands = descendantsOfType(root, 'command'); + expect(commands).toHaveLength(2); + const argv = commands.map((cmd) => + cmd.namedChildren.map((part) => (part.type === 'command_name' ? part.namedChildren[0]!.text : part.text)), + ); + expect(argv).toEqual([ + ['git', 'status'], + ['rm', '-rf', '/'], + ]); + }); +}); + +describe('offsets', () => { + it('uses UTF-16 code unit offsets', () => { + // 🎉 is two UTF-16 code units. + const { rootNode } = parseOk('x🎉y; ls'); + expect(rootNode.endIndex).toBe(8); + const commands = descendantsOfType(rootNode, 'command'); + expect(commands).toHaveLength(2); + expect(commands[0]!.startIndex).toBe(0); + expect(commands[0]!.endIndex).toBe(4); + expect(commands[1]!.startIndex).toBe(6); + expect(commands[1]!.endIndex).toBe(8); + expect(commands[1]!.text).toBe('ls'); + }); + + it('node.text always equals the source slice', () => { + const source = 'FOO=bar env | grep FOO && echo "ok $X"'; + const { rootNode } = parseOk(source); + const walk = (node: SyntaxNode): void => { + expect(node.text).toBe(source.slice(node.startIndex, node.endIndex)); + node.children.forEach(walk); + }; + walk(rootNode); + }); +}); + +describe('adversarial input', () => { + it('recovers unterminated quotes, substitutions and expansions', () => { + for (const source of ['echo "abc', "echo 'abc", 'echo $(foo', 'echo ${foo', 'echo `foo']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('recovers stray operators without throwing', () => { + for (const source of ['echo )', '&& echo', '| echo', ')']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('parses a 500KB double-quoted string under the default budget', () => { + // Regression: per-character budget.tick() used to burn the 50k node cap + // on long strings even though only a handful of nodes are created. + const body = 'a'.repeat(500 * 1024); + const result = parse(`echo "${body}"`); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(false); + const content = descendantsOfType(result.rootNode, 'string_content'); + expect(content).toHaveLength(1); + expect(content[0]!.text).toBe(body); + }); + + it('parses a 500KB heredoc body under the default budget', () => { + const body = 'b'.repeat(500 * 1024); + const result = parse(`cat < { + const source = 'echo ' + '${a:-'.repeat(1000) + 'x' + '}'.repeat(1000); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles 1000-level nested command substitutions within the depth cap', () => { + const source = '$('.repeat(1000) + 'x' + ')'.repeat(1000); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles 1000-level nested subshells within the depth cap', () => { + const source = '('.repeat(1000) + 'x' + ')'.repeat(1000); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('parses a multi-megabyte single word without throwing', () => { + const source = 'a'.repeat(2 * 1024 * 1024); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(false); + const word = result.rootNode.namedChildren[0]!.namedChildren[0]!.namedChildren[0]!; + expect(word.type).toBe('word'); + expect(word.text).toBe(source); + }); + + it('does not throw on random binary garbage', () => { + let source = ''; + for (let i = 0; i < 20_000; i++) { + source += String.fromCodePoint(Math.floor(Math.random() * 0xd800)); + } + // The call itself must never throw; either it parses (possibly with + // errors) or the budget cuts it off. + const result = parse(source); + if (result.ok) { + expect(result.rootNode.type).toBe('program'); + } else { + expect(result.reason).toBe('aborted'); + } + }); + + it('does not throw on NUL bytes', () => { + const result = parse('echo \0\0 test \0'); + expect(result.ok).toBe(true); + }); +}); diff --git a/packages/tree-sitter-bash/test/parser-compound.test.ts b/packages/tree-sitter-bash/test/parser-compound.test.ts new file mode 100644 index 0000000000..a411fb4236 --- /dev/null +++ b/packages/tree-sitter-bash/test/parser-compound.test.ts @@ -0,0 +1,1232 @@ +import { describe, expect, it } from 'vitest'; + +import type { SyntaxNode } from '#/node'; +import { descendantsOfType } from '#/node'; +import { parse } from '#/parse'; + +/** Compact S-expression of the tree: named leaves are (type "text"), + * anonymous leaves are just their quoted type, inner nodes nest. */ +function sexp(node: SyntaxNode): string { + if (node.children.length === 0) { + return node.isNamed ? `(${node.type} ${JSON.stringify(node.text)})` : JSON.stringify(node.type); + } + const label = node.isNamed ? node.type : JSON.stringify(node.type); + return `(${label} ${node.children.map(sexp).join(' ')})`; +} + +function parseOk(source: string): { rootNode: SyntaxNode; hasError: boolean } { + const result = parse(source); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error('unreachable'); + return result; +} + +/** Parse and assert the exact tree shape plus the hasError flag. */ +function expectTree(source: string, expected: string, hasError = false): SyntaxNode { + const { rootNode, hasError: actual } = parseOk(source); + expect(actual).toBe(hasError); + expect(sexp(rootNode)).toBe(expected); + return rootNode; +} + +describe('if / while / until', () => { + it('parses a one-line if statement', () => { + expectTree( + 'if true; then echo yes; fi', + `(program (if_statement "if" (command (command_name (word "true"))) ";" "then" (command (command_name (word "echo")) (word "yes")) ";" "fi"))`, + ); + }); + + it('parses elif and else clauses', () => { + expectTree( + 'if a; then b; elif c; then d; else e; fi', + `(program (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" (elif_clause "elif" (command (command_name (word "c"))) ";" "then" (command (command_name (word "d"))) ";") (else_clause "else" (command (command_name (word "e"))) ";") "fi"))`, + ); + }); + + it('parses a multiline if without emitting newline nodes', () => { + expectTree( + 'if a\nthen\n b\nfi', + `(program (if_statement "if" (command (command_name (word "a"))) "then" (command (command_name (word "b"))) "fi"))`, + ); + }); + + it('parses a while loop with a trailing redirect', () => { + expectTree( + 'while read -r line; do echo "$line"; done < file', + `(program (redirected_statement (while_statement "while" (command (command_name (word "read")) (word "-r") (word "line")) ";" (do_group "do" (command (command_name (word "echo")) (string "\\"" (simple_expansion "$" (variable_name "line")) "\\"")) ";" "done")) (file_redirect "<" (word "file"))))`, + ); + }); + + it('parses until as a while_statement', () => { + expectTree( + 'until x; do y; done', + `(program (while_statement "until" (command (command_name (word "x"))) ";" (do_group "do" (command (command_name (word "y"))) ";" "done")))`, + ); + }); + + it('parses an empty do_group', () => { + expectTree( + 'while x; do done', + `(program (while_statement "while" (command (command_name (word "x"))) ";" (do_group "do" "done")))`, + ); + }); + + it('only treats keywords as keywords in statement position', () => { + expectTree( + 'echo if for while', + `(program (command (command_name (word "echo")) (word "if") (word "for") (word "while")))`, + ); + expectTree('fi', `(program (command (command_name (word "fi"))))`); + expectTree('done', `(program (command (command_name (word "done"))))`); + expectTree('}', `(program (command (command_name (word "}"))))`); + expectTree('time ls -la', `(program (command (command_name (word "time")) (word "ls") (word "-la")))`); + // Reserved words are plain arguments inside conditions and bodies. + expectTree( + 'if echo then; then x; fi', + `(program (if_statement "if" (command (command_name (word "echo")) (word "then")) ";" "then" (command (command_name (word "x"))) ";" "fi"))`, + ); + // A prefix assignment disables the keyword reading. + expectTree( + 'x=1 if a; then b; fi', + `(program (command (variable_assignment (variable_name "x") "=" (number "1")) (command_name (word "if")) (word "a")) ";" (command (command_name (word "then")) (word "b")) ";" (command (command_name (word "fi"))))`, + ); + }); +}); + +describe('for / select', () => { + it('parses a for-in loop', () => { + expectTree( + 'for f in a b c; do echo "$f"; done', + `(program (for_statement "for" (variable_name "f") "in" (word "a") (word "b") (word "c") ";" (do_group "do" (command (command_name (word "echo")) (string "\\"" (simple_expansion "$" (variable_name "f")) "\\"")) ";" "done")))`, + ); + }); + + it('parses a for loop without in', () => { + expectTree( + 'for f; do x; done', + `(program (for_statement "for" (variable_name "f") ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses select as a for_statement', () => { + expectTree( + 'select opt in a b; do echo $opt; done', + `(program (for_statement "select" (variable_name "opt") "in" (word "a") (word "b") ";" (do_group "do" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "opt"))) ";" "done")))`, + ); + }); + + it('parses newline-separated for headers', () => { + expectTree( + 'for f in a b\ndo x\ndone', + `(program (for_statement "for" (variable_name "f") "in" (word "a") (word "b") (do_group "do" (command (command_name (word "x"))) "done")))`, + ); + }); + + it('flags a bare `in` with no values as an error', () => { + expectTree( + 'select x in; do y; done', + `(program (for_statement "select" (variable_name "x") (ERROR "in") ";" (do_group "do" (command (command_name (word "y"))) ";" "done")))`, + true, + ); + }); + + it('parses a c-style for loop', () => { + expectTree( + 'for ((i=0;i<3;i++)); do echo $i; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "))" ";" (do_group "do" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "i"))) ";" "done")))`, + ); + }); + + it('parses a spaced c-style for with an update assignment operator', () => { + expectTree( + 'for (( i = 0; i < 10; i += 2 )); do x; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) ";" (binary_expression (word "i") "<" (number "10")) ";" (binary_expression (word "i") "+=" (number "2")) "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses an empty c-style for header', () => { + expectTree( + 'for ((;;)); do x; done', + `(program (c_style_for_statement "for" "((" ";" ";" "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses comma-separated c-style for parts', () => { + expectTree( + 'for ((i=0,j=1; i<3; i++,j--)); do x; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) "," (variable_assignment (variable_name "j") "=" (number "1")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "," (postfix_expression (word "j") "--") "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses a compound_statement as the c-style for body', () => { + expectTree( + 'for ((i=0;i<3;i++)) { echo $i; }', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "))" (compound_statement "{" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "i"))) ";" "}")))`, + ); + }); +}); + +describe('case', () => { + it('parses a multi-item case with all termination forms', () => { + expectTree( + 'case $x in\n a) echo A ;;\n b|c) echo BC ;&\n *) echo other ;;\nesac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (word "a") ")" (command (command_name (word "echo")) (word "A")) ";;") (case_item (extglob_pattern "b") "|" (word "c") ")" (command (command_name (word "echo")) (word "BC")) ";&") (case_item (extglob_pattern "*") ")" (command (command_name (word "echo")) (word "other")) ";;") "esac"))`, + ); + }); + + it('parses optional parens and empty item bodies', () => { + expectTree( + 'case $x in (a) x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item "(" (word "a") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + expectTree( + 'case x in a) ;; esac', + `(program (case_statement "case" (word "x") "in" (case_item (word "a") ")" ";;") "esac"))`, + ); + expectTree( + 'case $x in esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" "esac"))`, + ); + }); + + it('parses pattern forms: globs, quotes and expansions', () => { + expectTree( + 'case $x in [a-z]) x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (extglob_pattern "[a-z]") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + expectTree( + 'case $x in "a") x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (string "\\"" (string_content "a") "\\"") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + expectTree( + 'case $x in $v|${w}) x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (simple_expansion "$" (variable_name "v")) "|" (expansion "\${" (variable_name "w") "}") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + }); + + it('ends the last case_item at esac even without ;;', () => { + expectTree( + 'case $x in a) x esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (word "a") ")" (command (command_name (word "x")))) "esac"))`, + ); + }); + + it('flags a fallthrough terminator on the last case_item', () => { + const { hasError } = parseOk('case $x in a) x ;;& esac'); + expect(hasError).toBe(true); + }); +}); + +describe('functions and compound statements', () => { + it('parses the name() form', () => { + expectTree( + 'foo() { echo hi; }', + `(program (function_definition (word "foo") "(" ")" (compound_statement "{" (command (command_name (word "echo")) (word "hi")) ";" "}")))`, + ); + }); + + it('parses the function keyword forms', () => { + expectTree( + 'function bar { echo hi; }', + `(program (function_definition "function" (word "bar") (compound_statement "{" (command (command_name (word "echo")) (word "hi")) ";" "}")))`, + ); + expectTree( + 'function baz() { echo hi; return 0; }', + `(program (function_definition "function" (word "baz") "(" ")" (compound_statement "{" (command (command_name (word "echo")) (word "hi")) ";" (command (command_name (word "return")) (number "0")) ";" "}")))`, + ); + expectTree( + 'f () { x; }', + `(program (function_definition (word "f") "(" ")" (compound_statement "{" (command (command_name (word "x"))) ";" "}")))`, + ); + }); + + it('parses subshell, test and if bodies', () => { + expectTree( + 'foo() ( echo sub )', + `(program (function_definition (word "foo") "(" ")" (subshell "(" (command (command_name (word "echo")) (word "sub")) ")")))`, + ); + expectTree( + 'foo() [[ -f x ]]', + `(program (function_definition (word "foo") "(" ")" (test_command "[[" (unary_expression (test_operator "-f") (word "x")) "]]")))`, + ); + expectTree( + 'foo() if a; then b; fi', + `(program (function_definition (word "foo") "(" ")" (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi")))`, + ); + }); + + it('keeps a trailing redirect inside the function_definition', () => { + expectTree( + 'foo() { x; } > out', + `(program (function_definition (word "foo") "(" ")" (compound_statement "{" (command (command_name (word "x"))) ";" "}") (file_redirect ">" (word "out"))))`, + ); + }); + + it('parses a standalone compound statement', () => { + expectTree( + '{ echo a; echo b; }', + `(program (compound_statement "{" (command (command_name (word "echo")) (word "a")) ";" (command (command_name (word "echo")) (word "b")) ";" "}"))`, + ); + expectTree('{ls;}', `(program (compound_statement "{" (command (command_name (word "ls"))) ";" "}"))`); + }); + + it('parses coproc as an ordinary command (like the reference)', () => { + expectTree( + 'coproc myjob { echo hi; }', + `(program (command (command_name (word "coproc")) (word "myjob") (word "{") (word "echo") (word "hi")) ";" (command (command_name (word "}"))))`, + ); + }); +}); + +describe('test commands', () => { + it('parses unary test operators', () => { + expectTree( + '[[ -f file.txt ]]', + `(program (test_command "[[" (unary_expression (test_operator "-f") (word "file.txt")) "]]"))`, + ); + expectTree( + '[[ -z $s ]]', + `(program (test_command "[[" (unary_expression (test_operator "-z") (simple_expansion "$" (variable_name "s"))) "]]"))`, + ); + }); + + it('parses string comparisons', () => { + expectTree( + '[[ $x == "foo" ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (string "\\"" (string_content "foo") "\\"")) "]]"))`, + ); + expectTree( + '[ "$a" = "b" ]', + `(program (test_command "[" (binary_expression (string "\\"" (simple_expansion "$" (variable_name "a")) "\\"") "=" (string "\\"" (string_content "b") "\\"")) "]"))`, + ); + }); + + it('parses =~ with a regex right side', () => { + expectTree( + '[[ $x =~ ^ab+c$ ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "=~" (regex "^ab+c$")) "]]"))`, + ); + expectTree( + '[[ $x =~ a|b ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "=~" (regex "a|b")) "]]"))`, + ); + expectTree( + '[[ $x =~ $re ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "=~" (simple_expansion "$" (variable_name "re"))) "]]"))`, + ); + }); + + it('parses pattern right sides: regex after =, extglob_pattern after ==/!=', () => { + expectTree( + '[[ a = b*c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b*c")) "]]"))`, + ); + expectTree( + '[[ a = bc ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (word "bc")) "]]"))`, + ); + expectTree( + '[[ $x == b*c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (extglob_pattern "b*c")) "]]"))`, + ); + expectTree( + '[[ $x == 123 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (number "123")) "]]"))`, + ); + }); + + it('parses && / || / ! combinations', () => { + expectTree( + '[[ -n $s && -d /tmp ]]', + `(program (test_command "[[" (binary_expression (unary_expression (test_operator "-n") (simple_expansion "$" (variable_name "s"))) "&&" (unary_expression (test_operator "-d") (word "/tmp"))) "]]"))`, + ); + expectTree( + '[[ a < b || ! -e f ]]', + `(program (test_command "[[" (binary_expression (binary_expression (word "a") "<" (word "b")) "||" (unary_expression "!" (unary_expression (test_operator "-e") (word "f")))) "]]"))`, + ); + }); + + it('parses binary test operators and parentheses', () => { + expectTree( + '[[ $a -eq 3 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) (test_operator "-eq") (number "3")) "]]"))`, + ); + expectTree( + '[[ $x -eq 1 && $y -ne 2 ]]', + `(program (test_command "[[" (binary_expression (binary_expression (simple_expansion "$" (variable_name "x")) (test_operator "-eq") (number "1")) "&&" (binary_expression (simple_expansion "$" (variable_name "y")) (test_operator "-ne") (number "2"))) "]]"))`, + ); + // Inside parentheses the == right side is a word, not extglob_pattern. + expectTree( + '[[ (a == b) && c ]]', + `(program (test_command "[[" (binary_expression (parenthesized_expression "(" (binary_expression (word "a") "==" (word "b")) ")") "&&" (word "c")) "]]"))`, + ); + }); + + it('parses the single-bracket form and adjacent =', () => { + expectTree('[ -f file ]', `(program (test_command "[" (unary_expression (test_operator "-f") (word "file")) "]"))`); + expectTree('[ x ]', `(program (test_command "[" (word "x") "]"))`); + expectTree('[[ a=b ]]', `(program (test_command "[[" (word "a=b") "]]"))`); + }); + + it('parses negated test commands', () => { + expectTree( + '! [[ -f x ]]', + `(program (negated_command "!" (test_command "[[" (unary_expression (test_operator "-f") (word "x")) "]]")))`, + ); + expectTree( + '! [ -f x ]', + `(program (negated_command "!" (test_command "[" (unary_expression (test_operator "-f") (word "x")) "]")))`, + ); + }); + + it('keeps [ in argument position as plain words', () => { + expectTree('echo [ x ]', `(program (command (command_name (word "echo")) (word "[") (word "x") (word "]")))`); + }); + + it('recovers an unterminated test command with a zero-width closer', () => { + expectTree('[ -f file', `(program (test_command "[" (unary_expression (test_operator "-f") (word "file")) "]"))`, true); + }); +}); + +describe('arithmetic', () => { + it('parses operator precedence levels', () => { + expectTree( + 'echo $((1 + 2 * 3))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (number "1") "+" (binary_expression (number "2") "*" (number "3"))) "))")))`, + ); + expectTree( + 'echo $((x << 2 | 1))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (variable_name "x") "<<" (number "2")) "|" (number "1")) "))")))`, + ); + expectTree( + 'echo $((n % 2 == 0))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (variable_name "n") "%" (number "2")) "==" (number "0")) "))")))`, + ); + }); + + it('parses ternary, postfix, prefix and unary expressions', () => { + expectTree( + 'echo $((a > b ? a : b))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (ternary_expression (binary_expression (variable_name "a") ">" (variable_name "b")) "?" (variable_name "a") ":" (variable_name "b")) "))")))`, + ); + expectTree( + 'echo $((i++ + --j))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (postfix_expression (variable_name "i") "++") "+" (unary_expression "--" (variable_name "j"))) "))")))`, + ); + // Prefix operators grab the whole tighter-precedence expression, like + // the reference: -(x + ~y). + expectTree( + 'echo $((-x + ~y))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (unary_expression "-" (binary_expression (variable_name "x") "+" (unary_expression "~" (variable_name "y")))) "))")))`, + ); + }); + + it('parses parentheses and exponent (left-associative in arithmetic)', () => { + expectTree( + 'echo $(( (1+2) ** 3 ))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (parenthesized_expression "(" (binary_expression (number "1") "+" (number "2")) ")") "**" (number "3")) "))")))`, + ); + expectTree( + 'echo $((2 ** 3 ** 2))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (number "2") "**" (number "3")) "**" (number "2")) "))")))`, + ); + }); + + it('parses assignment operators, subscripts and comma lists', () => { + expectTree( + 'echo $((x += 5))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (variable_name "x") "+=" (number "5")) "))")))`, + ); + expectTree( + 'echo $((x = y = 1))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (variable_name "x") "=" (variable_name "y")) "=" (number "1")) "))")))`, + ); + expectTree( + 'echo $((arr[0] + 1))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (subscript (variable_name "arr") "[" (number "0") "]") "+" (number "1")) "))")))`, + ); + expectTree( + 'echo $((1, 2, 3))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (number "1") "," (number "2") "," (number "3") "))")))`, + ); + }); + + it('parses the ((…)) command and legacy $[…] form', () => { + expectTree( + '((x = y + 1))', + `(program (command (command_name (arithmetic_expansion "((" (binary_expression (variable_name "x") "=" (binary_expression (variable_name "y") "+" (number "1"))) "))"))))`, + ); + expectTree( + '(( ls ))', + `(program (command (command_name (arithmetic_expansion "((" (variable_name "ls") "))"))))`, + ); + expectTree( + 'echo $[1+2]', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$[" (binary_expression (number "1") "+" (number "2")) "]")))`, + ); + }); + + it('parses arithmetic across newlines and nested arithmetic', () => { + expectTree( + 'echo $((\n1 +\n2\n))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (number "1") "+" (number "2")) "))")))`, + ); + expectTree( + 'for ((i = $((1 + 1)); i < 3; i++)); do x; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (arithmetic_expansion "$((" (binary_expression (number "1") "+" (number "1")) "))")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); +}); + +describe('arrays and subscripts', () => { + it('parses array assignments', () => { + expectTree( + 'arr=(a b "c d")', + `(program (variable_assignment (variable_name "arr") "=" (array "(" (word "a") (word "b") (string "\\"" (string_content "c d") "\\"") ")")))`, + ); + expectTree( + 'x+=(c d)', + `(program (variable_assignment (variable_name "x") "+=" (array "(" (word "c") (word "d") ")")))`, + ); + }); + + it('parses subscript assignments', () => { + expectTree( + 'arr[0]=x', + `(program (variable_assignment (subscript (variable_name "arr") "[" (number "0") "]") "=" (word "x")))`, + ); + expectTree( + 'a[i+1]=$y', + `(program (variable_assignment (subscript (variable_name "a") "[" (word "i+1") "]") "=" (simple_expansion "$" (variable_name "y"))))`, + ); + expectTree( + 'x[1]+=2', + `(program (variable_assignment (subscript (variable_name "x") "[" (number "1") "]") "+=" (number "2")))`, + ); + }); + + it('parses subscript expansions including @ and $ indexes', () => { + expectTree( + 'echo ${arr[@]}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "arr") "[" (word "@") "]") "}")))`, + ); + expectTree( + 'echo ${a[$i]}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "a") "[" (simple_expansion "$" (variable_name "i")) "]") "}")))`, + ); + }); +}); + +describe('declaration and unset commands', () => { + it('parses export/declare/local/readonly', () => { + expectTree( + 'export FOO=bar BAZ', + `(program (declaration_command "export" (variable_assignment (variable_name "FOO") "=" (word "bar")) (variable_name "BAZ")))`, + ); + expectTree( + 'declare -r x=1', + `(program (declaration_command "declare" (word "-r") (variable_assignment (variable_name "x") "=" (number "1"))))`, + ); + expectTree('local y', `(program (declaration_command "local" (variable_name "y")))`); + expectTree( + 'readonly z=2', + `(program (declaration_command "readonly" (variable_assignment (variable_name "z") "=" (number "2"))))`, + ); + }); + + it('parses unset', () => { + expectTree('unset a b', `(program (unset_command "unset" (variable_name "a") (variable_name "b")))`); + }); +}); + +describe('strings and brace expressions', () => { + it('parses ANSI-C strings as a single node', () => { + expectTree( + `echo $'a\\nb\\t'`, + `(program (command (command_name (word "echo")) (ansi_c_string "$'a\\\\nb\\\\t'")))`, + ); + }); + + it('parses $"…" as an anonymous $ plus a string (no translated_string)', () => { + expectTree( + 'echo $"hello $USER"', + `(program (command (command_name (word "echo")) "$" (string "\\"" (string_content "hello ") (simple_expansion "$" (variable_name "USER")) "\\"")))`, + ); + }); + + it('parses $"…" as translated_string outside argument position', () => { + expectTree( + 'x=$"v"', + `(program (variable_assignment (variable_name "x") "=" (translated_string "$" (string "\\"" (string_content "v") "\\""))))`, + ); + expectTree( + 'cat > $"out"', + `(program (redirected_statement (command (command_name (word "cat"))) (file_redirect ">" (translated_string "$" (string "\\"" (string_content "out") "\\"")))))`, + ); + expectTree( + 'for f in $"a"; do x; done', + `(program (for_statement "for" (variable_name "f") "in" (translated_string "$" (string "\\"" (string_content "a") "\\"")) ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + // Mid-concatenation the bare $ and the string stay separate pieces. + expectTree( + 'echo a$"b"', + `(program (command (command_name (word "echo")) (concatenation (word "a") "$" (string "\\"" (string_content "b") "\\""))))`, + ); + }); + + it('parses {N..M} brace expressions', () => { + expectTree( + 'echo {1..10}', + `(program (command (command_name (word "echo")) (brace_expression "{" (number "1") ".." (number "10") "}")))`, + ); + expectTree( + 'echo a{1..3}b', + `(program (command (command_name (word "echo")) (concatenation (word "a") (brace_expression "{" (number "1") ".." (number "3") "}") (word "b"))))`, + ); + expectTree( + 'x={1..5}', + `(program (variable_assignment (variable_name "x") "=" (brace_expression "{" (number "1") ".." (number "5") "}")))`, + ); + expectTree( + '{1..3}', + `(program (command (command_name (brace_expression "{" (number "1") ".." (number "3") "}"))))`, + ); + }); + + it('parses other brace forms as plain concatenations (like the reference)', () => { + expectTree( + 'echo {1..10..2}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "1..10..2") (word "}"))))`, + ); + expectTree( + 'echo {a..z}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "a..z") (word "}"))))`, + ); + expectTree( + 'echo {-5..5}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "-5..5") (word "}"))))`, + ); + expectTree( + 'echo a{b,c}d', + `(program (command (command_name (word "echo")) (concatenation (word "a") (word "{") (word "b,c") (word "}") (word "d"))))`, + ); + }); +}); + +describe('expansion operators', () => { + it('parses removal patterns as regex nodes', () => { + expectTree( + 'echo ${x##*/} ${x%%.*}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "##" (regex "*/") "}") (expansion "\${" (variable_name "x") "%%" (regex ".*") "}")))`, + ); + expectTree( + 'echo ${x##a b}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "##" (regex "a b") "}")))`, + ); + expectTree( + 'echo ${x#"pat"}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "#" (string "\\"" (string_content "pat") "\\"") "}")))`, + ); + }); + + it('parses replacements with regex pattern and literal replacement', () => { + expectTree( + 'echo ${x/y/z} ${x//a/b}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "y") "/" (word "z") "}") (expansion "\${" (variable_name "x") "//" (regex "a") "/" (word "b") "}")))`, + ); + expectTree( + 'echo ${x/a b/c}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "a b") "/" (word "c") "}")))`, + ); + expectTree( + 'echo ${x/a/$v}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "a") "/" (simple_expansion "$" (variable_name "v")) "}")))`, + ); + expectTree( + 'echo ${x/pat/rep/extra}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "pat") "/" (word "rep/extra") "}")))`, + ); + }); + + it('parses case modification and transformation operators', () => { + expectTree( + 'echo ${x^} ${x,,}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "^" "}") (expansion "\${" (variable_name "x") ",," "}")))`, + ); + expectTree( + 'echo ${x@Q}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "@" "Q" "}")))`, + ); + }); + + it('parses max-length expansions with arithmetic values', () => { + expectTree( + 'echo ${v:1:2}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":" (number "1") ":" (number "2") "}")))`, + ); + expectTree( + 'echo ${arr[@]:1:2}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "arr") "[" (word "@") "]") ":" (number "1") ":" (number "2") "}")))`, + ); + }); + + it('parses default values as words, splitting bare values with spaces', () => { + expectTree( + 'echo ${v:-1} ${v:=2x}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (word "1") "}") (expansion "\${" (variable_name "v") ":=" (word "2x") "}")))`, + ); + expectTree( + 'echo ${x:-d e f}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") ":-" (concatenation (word "d") (word " e f")) "}")))`, + ); + expectTree( + 'echo ${v:-"d e"}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (string "\\"" (string_content "d e") "\\"") "}")))`, + ); + }); + + it('parses indirect expansions with trailing * and @', () => { + expectTree( + 'echo ${!prefix*} ${!name@}', + `(program (command (command_name (word "echo")) (expansion "\${" "!" (variable_name "prefix") "*" "}") (expansion "\${" "!" (variable_name "name") "@" "}")))`, + ); + }); +}); + +describe('combinations', () => { + it('nests if inside for inside case', () => { + expectTree( + 'case $1 in start) for s in a b; do if [[ -n $s ]]; then echo "$s"; fi; done ;; *) echo usage ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "1")) "in" (case_item (word "start") ")" (for_statement "for" (variable_name "s") "in" (word "a") (word "b") ";" (do_group "do" (if_statement "if" (test_command "[[" (unary_expression (test_operator "-n") (simple_expansion "$" (variable_name "s"))) "]]") ";" "then" (command (command_name (word "echo")) (string "\\"" (simple_expansion "$" (variable_name "s")) "\\"")) ";" "fi") ";" "done")) ";;") (case_item (extglob_pattern "*") ")" (command (command_name (word "echo")) (word "usage")) ";;") "esac"))`, + ); + }); + + it('parses a heredoc inside a function body', () => { + expectTree( + 'foo() { cat < { + expectTree( + 'if a; then b; fi | grep x', + `(program (pipeline (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") "|" (command (command_name (word "grep")) (word "x"))))`, + ); + expectTree( + 'if a; then b; fi && c', + `(program (list (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") "&&" (command (command_name (word "c")))))`, + ); + expectTree( + 'echo hi | if a; then b; fi', + `(program (pipeline (command (command_name (word "echo")) (word "hi")) "|" (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi")))`, + ); + }); + + it('parses redirects on compound statements', () => { + expectTree( + 'if a; then b; fi > out', + `(program (redirected_statement (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") (file_redirect ">" (word "out"))))`, + ); + }); + + it('parses test commands as loop conditions', () => { + expectTree( + 'while [[ -n $s ]]; do s=; done', + `(program (while_statement "while" (test_command "[[" (unary_expression (test_operator "-n") (simple_expansion "$" (variable_name "s"))) "]]") ";" (do_group "do" (variable_assignment (variable_name "s") "=") ";" "done")))`, + ); + expectTree( + 'if [ -f f ]; then x; fi', + `(program (if_statement "if" (test_command "[" (unary_expression (test_operator "-f") (word "f")) "]") ";" "then" (command (command_name (word "x"))) ";" "fi"))`, + ); + }); + + it('parses negation, lists and nested ifs in conditions', () => { + expectTree( + 'if ! grep -q x f; then echo missing; fi', + `(program (if_statement "if" (negated_command "!" (command (command_name (word "grep")) (word "-q") (word "x") (word "f"))) ";" "then" (command (command_name (word "echo")) (word "missing")) ";" "fi"))`, + ); + expectTree( + 'if a && b; then c; fi', + `(program (if_statement "if" (list (command (command_name (word "a"))) "&&" (command (command_name (word "b")))) ";" "then" (command (command_name (word "c"))) ";" "fi"))`, + ); + expectTree( + 'if if a; then b; fi; then c; fi', + `(program (if_statement "if" (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") ";" "then" (command (command_name (word "c"))) ";" "fi"))`, + ); + }); + + it('parses arrays and subscript expansions inside functions', () => { + expectTree( + 'f() { arr=(1 2); echo ${arr[0]}; }', + `(program (function_definition (word "f") "(" ")" (compound_statement "{" (variable_assignment (variable_name "arr") "=" (array "(" (number "1") (number "2") ")")) ";" (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "arr") "[" (number "0") "]") "}")) ";" "}")))`, + ); + }); + + it('parses a command subshell argument and command substitutions in for values', () => { + expectTree( + 'foo (ls)', + `(program (command (command_name (word "foo")) (subshell "(" (command (command_name (word "ls"))) ")")))`, + ); + expectTree( + 'for i in $(seq 3); do echo $i; done', + `(program (for_statement "for" (variable_name "i") "in" (command_substitution "$(" (command (command_name (word "seq")) (number "3")) ")") ";" (do_group "do" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "i"))) ";" "done")))`, + ); + }); +}); + +describe('recovery and adversarial input', () => { + it('recovers unterminated compound commands without throwing', () => { + for (const source of [ + 'if a; then b', + 'if a', + 'while x; do y', + 'for f in a b; do x', + 'case $x in a) x', + 'case $x', + 'foo() { x;', + '{ x;', + 'function f', + 'for ((i=0;i<3', + 'echo $((1 + ', + '[[ -f x', + 'arr=(1 2', + ]) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('recovers stray case terminators at top level', () => { + for (const source of ['a ;& b', 'a ;;& b', ';;&']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('flags an empty subshell', () => { + const result = parse('()'); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles deeply nested compound commands within the depth cap', () => { + const source = 'if a; then '.repeat(600) + 'b' + ' fi'.repeat(600); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles deeply nested arithmetic parentheses within the depth cap', () => { + const source = 'echo $(( ' + '('.repeat(600) + '1' + ')'.repeat(600) + ' ))'; + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles deeply nested case statements within the depth cap', () => { + const source = 'case x in a) '.repeat(600) + 'y' + ' ;; esac'.repeat(600); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('parses a pathological test command under budget', () => { + const result = parse('[[ ' + 'a && '.repeat(2000) + 'a ]]'); + // Either parses (with bounded work) or the budget cuts it off; never throws. + if (result.ok) { + expect(result.rootNode.type).toBe('program'); + } else { + expect(result.reason).toBe('aborted'); + } + }); +}); + +describe('M2 review regressions', () => { + it('keeps a full case_statement inside $() despite pattern parens', () => { + expectTree( + 'x=$(case y in a) 1;; esac)', + `(program (variable_assignment (variable_name "x") "=" (command_substitution "$(" (case_statement "case" (word "y") "in" (case_item (word "a") ")" (command (command_name (number "1"))) ";;") "esac") ")")))`, + ); + // Optional-paren item form stays balanced too. + expectTree( + 'x=$(case y in (b) 2;; esac)', + `(program (variable_assignment (variable_name "x") "=" (command_substitution "$(" (case_statement "case" (word "y") "in" (case_item "(" (word "b") ")" (command (command_name (number "2"))) ";;") "esac") ")")))`, + ); + expectTree( + 'diff <(case y in a) 1;; esac) other', + `(program (command (command_name (word "diff")) (process_substitution "<(" (case_statement "case" (word "y") "in" (case_item (word "a") ")" (command (command_name (number "1"))) ";;") "esac") ")") (word "other")))`, + ); + }); + + it('does not mistake an argument named case for a case_statement in $()', () => { + expectTree( + '$(echo case; ls)', + `(program (command (command_name (command_substitution "$(" (command (command_name (word "echo")) (word "case")) ";" (command (command_name (word "ls"))) ")"))))`, + ); + }); + + it('converts test right-hand sides by operator, glob and paren depth', () => { + expectTree( + '[[ ( $a == x* ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "x*")) ")") "]]"))`, + ); + expectTree( + '[[ ( $a = x* ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "=" (word "x*")) ")") "]]"))`, + ); + expectTree( + '[[ $x == b=c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (word "b=c")) "]]"))`, + ); + expectTree( + '[[ a = b=c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b=c")) "]]"))`, + ); + expectTree( + '[[ a = =b ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "=b")) "]]"))`, + ); + }); + + it('parses statement-position ((word++…)) as a test_command', () => { + expectTree('((a++))', `(program (test_command "((" (word "a++") "))"))`); + expectTree('((a--))', `(program (test_command "((" (word "a--") "))"))`); + expectTree( + '((a++ + b))', + `(program (test_command "((" (binary_expression (word "a++") "+" (word "b")) "))"))`, + ); + // Other ((…)) forms and non-statement positions stay arithmetic. + expectTree( + '((b + a++))', + `(program (command (command_name (arithmetic_expansion "((" (binary_expression (variable_name "b") "+" (postfix_expression (variable_name "a") "++")) "))"))))`, + ); + expectTree( + 'echo ((a++))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "((" (postfix_expression (variable_name "a") "++") "))")))`, + ); + expectTree( + '! ((a++))', + `(program (negated_command "!" (test_command "((" (word "a++") "))")))`, + ); + }); + + it('keeps unconsumed expression input in ERROR nodes', () => { + const arithmetic = parseOk('echo $((x[1][2]))'); + expect(arithmetic.hasError).toBe(true); + expect(sexp(arithmetic.rootNode)).toBe( + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (subscript (variable_name "x") "[" (number "1") "]") (ERROR "[2]") "))")))`, + ); + const test = parseOk('[[ $a =~ a b ]]'); + expect(test.hasError).toBe(true); + const errors = descendantsOfType(test.rootNode, 'ERROR'); + expect(errors).toHaveLength(1); + expect(errors[0]!.text).toContain('b'); + }); + + it('parses negative offsets in max-length expansions as numbers', () => { + expectTree( + 'echo ${x: -5}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") ":" (number "-5") "}")))`, + ); + expectTree( + 'echo ${x: -5:2}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") ":" (number "-5") ":" (number "2") "}")))`, + ); + }); + + it('parses $0 and ${0} as special_variable_name', () => { + expectTree( + 'echo $0 ${0} $1 ${10}', + `(program (command (command_name (word "echo")) (simple_expansion "$" (special_variable_name "0")) (expansion "\${" (special_variable_name "0") "}") (simple_expansion "$" (variable_name "1")) (expansion "\${" (variable_name "10") "}")))`, + ); + }); + + it('degrades locally on deep pattern→string→expansion nesting', () => { + let inner = '${a#""}'; + for (let k = 0; k < 650; k++) inner = `\${a#"${inner}"}`; + const result = parse(`echo ${inner}`, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(true); + // Local degradation: the tree keeps the outer expansions instead of + // collapsing into a single ERROR child under program. + expect(result.rootNode.children[0]!.type).not.toBe('ERROR'); + expect(descendantsOfType(result.rootNode, 'expansion').length).toBeGreaterThan(100); + }); + + it('treats = and ! as word characters at operand position in tests', () => { + expectTree('[[ a!=b ]]', `(program (test_command "[[" (word "a!=b") "]]"))`); + expectTree('[[ a=b ]]', `(program (test_command "[[" (word "a=b") "]]"))`); + expectTree( + '[[ !x = y ]]', + `(program (test_command "[[" (binary_expression (word "!x") "=" (word "y")) "]]"))`, + ); + expectTree( + '[[ $a==b* ]]', + `(program (test_command "[[" (concatenation (simple_expansion "$" (variable_name "a")) (word "==b*")) "]]"))`, + ); + // But they are operators at operator position, even when attached. + expectTree( + '[[ a ==b ]]', + `(program (test_command "[[" (binary_expression (word "a") "==" (extglob_pattern "b")) "]]"))`, + ); + expectTree( + '[[ a =b ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (word "b")) "]]"))`, + ); + expectTree( + '[[ $a =~^x ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "=~" (regex "^x")) "]]"))`, + ); + }); + + it('parses [[ ]] as a concatenation plus a zero-width closer (reference quirk)', () => { + expectTree( + '[[ ]]', + `(program (test_command "[[" (concatenation (word "]") (word "]")) "]]"))`, + true, + ); + }); + + it('parses bare numbers inside square brackets as number pieces', () => { + expectTree( + 'echo [1] a[2]b', + `(program (command (command_name (word "echo")) (concatenation (word "[") (number "1") (word "]")) (concatenation (word "a") (word "[") (number "2") (word "]") (word "b"))))`, + ); + expectTree( + 'echo a[-5]b', + `(program (command (command_name (word "echo")) (concatenation (word "a") (word "[") (number "-5") (word "]") (word "b"))))`, + ); + }); +}); + +describe('M2 review round 2 regressions', () => { + it('applies the scanner.c extglob_pattern rule to ==/!= right sides', () => { + // Single letter followed by whitespace → glob; followed by `)` → word. + expectTree( + '[[ $a == b ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "b")) "]]"))`, + ); + expectTree( + '[[ (a == b) && c ]]', + `(program (test_command "[[" (binary_expression (parenthesized_expression "(" (binary_expression (word "a") "==" (word "b")) ")") "&&" (word "c")) "]]"))`, + ); + expectTree( + '[[ ( $a == x ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "x")) ")") "]]"))`, + ); + // Multi-character all-letter words stay words, dots don't count. + expectTree( + '[[ $a == foo ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "foo")) "]]"))`, + ); + expectTree( + '[[ $a == x.y ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "x.y")) "]]"))`, + ); + // Any non-letter non-dot character makes it a glob. + for (const word of ['x1', 'foo-', 'x_', 'b*c=d', 'bc=d', 'b?=e', '.x', 'a-b']) { + const result = parseOk(`[[ $a == ${word} ]]`); + const globs = descendantsOfType(result.rootNode, 'extglob_pattern'); + expect(globs.map((g) => g.text)).toEqual([word]); + } + // `=` vetoes only as the second character; a leading `=` vetoes too. + expectTree( + '[[ $x == b=c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (word "b=c")) "]]"))`, + ); + expectTree( + '[[ a == =b ]]', + `(program (test_command "[[" (binary_expression (word "a") "==" (word "=b")) "]]"))`, + ); + // Round-1 fixed behavior must not regress. + expectTree( + '[[ $x == b*c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (extglob_pattern "b*c")) "]]"))`, + ); + expectTree( + '[[ $x == 123 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (number "123")) "]]"))`, + ); + expectTree( + '[[ a ==b ]]', + `(program (test_command "[[" (binary_expression (word "a") "==" (extglob_pattern "b")) "]]"))`, + ); + }); + + it('keeps the = right-side regex rule independent of the glob rule', () => { + expectTree( + '[[ a = b*c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b*c")) "]]"))`, + ); + expectTree( + '[[ a = bc ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (word "bc")) "]]"))`, + ); + expectTree( + '[[ a = b=c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b=c")) "]]"))`, + ); + expectTree( + '[[ ( $a = x* ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "=" (word "x*")) ")") "]]"))`, + ); + }); + + it('survives 2500 levels of lexer-side pattern nesting', () => { + let inner = '${a#""}'; + for (let k = 0; k < 2500; k++) inner = `\${a#"${inner}"}`; + const result = parse(`echo ${inner}`, { timeoutMs: 30_000 }); + // No RangeError, no abort: the scan depth cap degrades locally. + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(true); + expect(result.rootNode.children[0]!.type).not.toBe('ERROR'); + }); + + it('parses ((…)) inside a test as an arithmetic_expansion', () => { + expectTree( + '[[ ((a)) == x ]]', + `(program (test_command "[[" (binary_expression (arithmetic_expansion "((" (variable_name "a") "))") "==" (extglob_pattern "x")) "]]"))`, + ); + // Without the closing )), parens still nest as parenthesized_expressions. + expectTree( + '[[ ((a) == x) && y ]]', + `(program (test_command "[[" (binary_expression (parenthesized_expression "(" (binary_expression (parenthesized_expression "(" (word "a") ")") "==" (word "x")) ")") "&&" (word "y")) "]]"))`, + ); + }); + + it('parses ((ab-cd++)) as a test_command', () => { + expectTree('((ab-cd++))', `(program (test_command "((" (word "ab-cd++") "))"))`); + }); + + it('flags a missing separator before a compound keyword', () => { + for (const source of [ + 'if a; then case x in b) 1;; esac elif c; then d; fi', + 'if a; then b fi', + '{ ls }', + 'while x do y done', + 'for f in a b; do x done', + ]) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + // Valid inputs stay clean: case items need no terminator before esac, + // subshells none before ), compounds with proper terminators. + for (const source of ['if a; then b; fi', '{ ls; }', '(ls)', 'case $x in a) x esac', 'while x; do y; done']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(false); + } + }); +}); + +describe('M2 review round 3 regressions', () => { + it('converts […]-containing glob right sides as a whole', () => { + expectTree( + '[[ $ver == [0-9]* ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "ver")) "==" (extglob_pattern "[0-9]*")) "]]"))`, + ); + expectTree( + '[[ $a == [abc] ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "[abc]")) "]]"))`, + ); + expectTree( + '[[ ( $a == [xyz] ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "[xyz]")) ")") "]]"))`, + ); + expectTree( + '[[ $a == [!a]* ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "[!a]*")) "]]"))`, + ); + expectTree( + '[[ $a = [abc] ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "=" (regex "[abc]")) "]]"))`, + ); + // Bracket fragments outside test commands are untouched. + expectTree( + 'echo a[2]b', + `(program (command (command_name (word "echo")) (concatenation (word "a") (word "[") (number "2") (word "]") (word "b"))))`, + ); + }); + + it('keeps -word as a word when no operand follows it', () => { + expectTree( + '[[ $a == -foo ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "-foo")) "]]"))`, + ); + expectTree( + '[[ -foo == x ]]', + `(program (test_command "[[" (binary_expression (word "-foo") "==" (extglob_pattern "x")) "]]"))`, + ); + expectTree('[[ -f ]]', `(program (test_command "[[" (word "-f") "]]"))`); + // A real operand still makes it a test_operator. + expectTree( + '[[ -f file.txt ]]', + `(program (test_command "[[" (unary_expression (test_operator "-f") (word "file.txt")) "]]"))`, + ); + }); + + it('parses extglob group patterns after ==/!=', () => { + expectTree( + '[[ $a == +(!a) ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "+(!a)")) "]]"))`, + ); + expectTree( + '[[ $a == ?(a|b) ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "?(a|b)")) "]]"))`, + ); + expectTree( + '[[ $a == !(x) ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "!(x)")) "]]"))`, + ); + }); + + it('parses a negative number operand in tests as unary minus', () => { + expectTree( + '[[ $a == -1 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (unary_expression "-" (number "1"))) "]]"))`, + ); + expectTree( + '[[ $a -eq -1 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) (test_operator "-eq") (unary_expression "-" (number "1"))) "]]"))`, + ); + }); + + it('handles escaped characters in glob right sides', () => { + expectTree( + '[[ $a == foo\\ bar ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "foo\\\\ bar")) "]]"))`, + ); + expectTree( + '[[ $a == b\\*c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "b\\\\*c")) "]]"))`, + ); + }); + + it('extends elif/else clause ranges over the trailing newline', () => { + const { rootNode, hasError } = parseOk('if a; then b; elif c; then d\nelse e\nfi'); + expect(hasError).toBe(false); + const elifClause = descendantsOfType(rootNode, 'elif_clause')[0]!; + const elseClause = descendantsOfType(rootNode, 'else_clause')[0]!; + expect(elifClause.text).toBe('elif c; then d\n'); + expect(elseClause.text).toBe('else e\n'); + }); + + it('degrades 400-level command substitution nesting locally', () => { + const source = 'echo ' + '$('.repeat(400) + 'x' + ')'.repeat(400); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(true); + // Local degradation (MAX_SUBSTITUTION_DEPTH), not a last-resort + // single-ERROR root. + expect(result.rootNode.children[0]!.type).not.toBe('ERROR'); + expect(descendantsOfType(result.rootNode, 'command_substitution').length).toBeGreaterThan(100); + }); +}); diff --git a/packages/tree-sitter-bash/test/performance.test.ts b/packages/tree-sitter-bash/test/performance.test.ts new file mode 100644 index 0000000000..13b094a346 --- /dev/null +++ b/packages/tree-sitter-bash/test/performance.test.ts @@ -0,0 +1,95 @@ +// test/performance.test.ts +// +// Performance smoke tests — order-of-magnitude guards only (no absolute +// timing assertions beyond the documented abort path), meant to catch +// accidental quadratic complexity in future changes. + +import { describe, expect, it } from 'vitest'; + +import { parse } from '#/parse'; + +/** ~100 KB of realistic script-shaped text (functions, loops, redirects, + * expansions, heredocs — not a pathological single construct). */ +function realisticScript(): string { + const block = `deploy() { + local version="$1" + if [[ -z "\${version}" ]]; then + echo "usage: deploy " >&2 + return 1 + fi + for pkg in core web worker; do + tar -czf "dist/\${pkg}-\${version}.tar.gz" "build/\${pkg}" || return 2 + done + grep -q "release" CHANGELOG.md && echo "releasing \${version}" >> deploy.log + ssh deploy@example.com "mkdir -p /srv/app/releases/\${version}" < /dev/null + rsync -a --delete dist/ deploy@example.com:/srv/app/releases/\${version}/ | tee -a deploy.log +} + +while read -r name status; do + case "$status" in + ok) echo "\${name}: fine" ;; + *) echo "\${name}: \${status}" >&2 ;; + esac +done < services.txt + +cat <<'SCRIPT_EOF' +deploy finished at $(date) +SCRIPT_EOF + +`; + return block.repeat(Math.ceil(100_000 / block.length)); +} + +describe('performance smoke', () => { + it('parses a 100KB realistic script fast (default budget suffices)', () => { + const source = realisticScript(); + expect(source.length).toBeGreaterThan(100_000); + const start = performance.now(); + const result = parse(source); + const elapsed = performance.now() - start; + if (!result.ok) { + // An abort is acceptable only when it is prompt (< 100 ms). + expect(result).toEqual({ ok: false, reason: 'aborted' }); + expect(elapsed).toBeLessThan(100); + return; + } + // Completed within the default 50 ms budget; re-parse unbounded and + // assert the order of magnitude (< 1 s for 100KB) as a quadratic guard. + const restart = performance.now(); + const full = parse(source, { timeoutMs: Number.POSITIVE_INFINITY }); + const fullElapsed = performance.now() - restart; + expect(full.ok).toBe(true); + expect(fullElapsed).toBeLessThan(1000); + console.log(`100KB realistic script: default-budget parse ${elapsed.toFixed(1)}ms, unbounded ${fullElapsed.toFixed(1)}ms`); + }); + + it('the abort path is prompt (< 100 ms) on a node-budget bomb', () => { + const source = 'echo a; '.repeat(50_000); + const start = performance.now(); + const result = parse(source); + const elapsed = performance.now() - start; + expect(result).toEqual({ ok: false, reason: 'aborted' }); + expect(elapsed).toBeLessThan(100); + console.log(`abort after ${elapsed.toFixed(1)}ms on a 400KB node-budget bomb`); + }); + + it('typical one-line commands parse in well under a millisecond', () => { + // Warm up. + parse('git status && rm -rf /'); + const iterations = 200; + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + parse('git status && rm -rf /'); + } + const perParse = (performance.now() - start) / iterations; + expect(perParse).toBeLessThan(5); + console.log(`typical command: ${(perParse * 1000).toFixed(0)}µs per parse`); + }); + + it('a 500KB heredoc body parses within the default budget (few nodes)', () => { + const source = `cat <= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -7688,6 +7704,10 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-pty@1.1.0: resolution: {integrity: sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==} @@ -9155,6 +9175,14 @@ packages: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true + tree-sitter-bash@0.25.0: + resolution: {integrity: sha512-gZtlj9+qFS81qKxpLfD6H0UssQ3QBc/F0nKkPsiFDyfQF2YBqYvglFJUzchrPpVhZe9kLZTrJ9n2J6lmka69Vg==} + peerDependencies: + tree-sitter: ^0.25.0 + peerDependenciesMeta: + tree-sitter: + optional: true + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -9714,6 +9742,14 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-tree-sitter@0.25.10: + resolution: {integrity: sha512-Y09sF44/13XvgVKgO2cNDw5rGk6s26MgoZPXLESvMXeefBf7i6/73eFurre0IsTW6E14Y0ArIzhUMmjoc7xyzA==} + peerDependencies: + '@types/emscripten': ^1.40.0 + peerDependenciesMeta: + '@types/emscripten': + optional: true + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -17187,6 +17223,8 @@ snapshots: node-addon-api@7.1.1: {} + node-addon-api@8.9.0: {} + node-domexception@1.0.0: {} node-emoji@2.2.0: @@ -17208,6 +17246,8 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-gyp-build@4.8.4: {} + node-pty@1.1.0: dependencies: node-addon-api: 7.1.1 @@ -19004,6 +19044,11 @@ snapshots: tree-kill@1.2.2: {} + tree-sitter-bash@0.25.0: + dependencies: + node-addon-api: 8.9.0 + node-gyp-build: 4.8.4 + trim-lines@3.0.1: {} trough@2.2.0: {} @@ -19547,6 +19592,8 @@ snapshots: web-streams-polyfill@3.3.3: {} + web-tree-sitter@0.25.10: {} + webidl-conversions@3.0.1: {} webidl-conversions@7.0.0: