From 0ec395947685bd1ffb94fa8647711b9e9b68bb91 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 23:57:29 +0000 Subject: [PATCH] perf(lexer): a word position asks about zsh groups without measuring the file hellish -n 6000-function file: 630 ms (bash 5.3.9: 32 ms) after: 28 ms Issue #135: sourcing a 3.6k-line rc cost 181 ms against bash's 52 ms, "pure CPU". Under callgrind, 54% of every instruction retired while sourcing the hellishrc_plugins framework (3.5k lines) was strlen, all of it from one line: int xg_alt_group(const char *at) { return (xg_alt_group_n(at, at + ft_strlen(at), false)); } The lexer asks at every byte of every word whether a zsh alternation group `(a|b)` starts there, with `at` pointing into the whole input, so each question first measured the rest of the file -- quadratic in its size, in either dialect. xg_alt_group_n answers 0 unless *at is '(' and the zsh dialect is on; those two tests now come first and the end is measured only when they pass. Same answers, by construction. Instructions to source that framework: 457M -> 208M. OPT=1, min of 7: bash 5.3.9 develop this -n 6000 small funcs 32 ms 631 ms 28 ms source the framework 24 ms 100 ms 74 ms source 6000 trivial lines 16 ms 54 ms 24 ms What remains of #135 is linear and elsewhere: re-lexing a chunk each time an open construct grows past a heredoc line, the heredoc pre-scan lexing line by line, and the lexer's per-byte lookaheads. tests/parse_scaling_test.py gains a word-heavy case, 2000 vs 16000 lines via -n and via source, no fast pass: on develop's ASan build the ratios are 39.6 and 22.0 (quadratic), here 7.0 and 8.0 (linear, under the new 20.0 bound). The existing cases short-circuit anything under 1.5 s, which this quadratic stayed under at their sizes. Refs #135 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RAmeHfJNm7XjYbMNrQqvkG --- src/execution/case_match_ext3.c | 9 ++++++- tests/parse_scaling_test.py | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/execution/case_match_ext3.c b/src/execution/case_match_ext3.c index cb59cf4f..a3590289 100644 --- a/src/execution/case_match_ext3.c +++ b/src/execution/case_match_ext3.c @@ -82,9 +82,16 @@ int xg_alt_group_n(const char *at, const char *pe, bool brk) /* The same question about a whole NUL-terminated word, which is what the lexer, the word reparser and the filename globber all hold -- and which scans parens the way a lexer does, without the matcher's bracket rule, - so where a group ENDS for the grammar stays exactly where it was. */ + so where a group ENDS for the grammar stays exactly where it was. + The lexer asks this at every byte of every word, with `at` pointing + into the whole input: measuring the rest of it first made each question + cost the length of the file, and sourcing 3.5k lines spent half its time + in strlen (#135). Only a `(` in the zsh dialect can start a group, so the + end is found only then. */ int xg_alt_group(const char *at) { + if (*at != '(' || !glob_zsh()) + return (0); return (xg_alt_group_n(at, at + ft_strlen(at), false)); } diff --git a/tests/parse_scaling_test.py b/tests/parse_scaling_test.py index 3708b42f..cfcb6b96 100644 --- a/tests/parse_scaling_test.py +++ b/tests/parse_scaling_test.py @@ -69,6 +69,17 @@ def themes_like(n): return "".join(out) +def word_heavy(n): + """Many short functions, dense with words: every position of every word + asked whether a zsh alternation group starts there, and the question + first measured the rest of the input with strlen -- quadratic in the + size of the file, however it arrived (#135). 6000 such lines took + 630ms to parse where bash takes 32ms, and 16000 took seconds.""" + return "".join('f%d() { local a=$1 b="$2"; echo "$a" x y z ' + '| grep -q q && printf "%%s\\n" "$b"; }\n' % i + for i in range(n)) + + def timed(args, stdin_path=None, timeout=300): best = None for _ in range(2): @@ -117,6 +128,33 @@ def scaling_check(name, gen, via_stdin=False, via_source=False): check(name, t1 <= ABS_CAP and t4 / max(t1, 0.02) < RATIO_MAX, detail) +WORD_N = 2000 +WORD_RATIO_MAX = 20.0 # an 8x size step: linear ~8, quadratic ~64 + + +def word_scaling_check(name, argv): + """t(8N) against t(N), no fast pass. The step is 8x, not 4x: under + ASan the linear part is heavy enough that a 4x step put the quadratic + at a ratio of ~10 -- too close to linear to call on a busy runner.""" + small = write_tmp(word_heavy(WORD_N)) + big = write_tmp(word_heavy(WORD_N * 8)) + try: + run = (lambda p: argv[:-1] + [argv[-1] + p]) if argv[-1] == ". " \ + else (lambda p: argv + [p]) + t1 = timed(run(small)) + t4 = timed(run(big)) + except subprocess.TimeoutExpired: + check(name, False, "timed out -- parse is grossly superlinear") + return + finally: + os.unlink(small) + os.unlink(big) + detail = "t(%d)=%.3fs t(%d)=%.3fs ratio=%.1f" % ( + WORD_N, t1, WORD_N * 8, t4, t4 / max(t1, 0.005)) + print(" " + name + ": " + detail) + check(name, t4 / max(t1, 0.01) < WORD_RATIO_MAX, detail) + + def main(): scaling_check("hazard compound via -n FILE", hazard_compound) scaling_check("hazard compound via piped stdin", hazard_compound, @@ -126,6 +164,12 @@ def main(): "hazards must not re-parse the open construct)", themes_like, via_source=True) + # Word-heavy text is cheap per line, so the quadratic stayed under the + # fast pass at the sizes above: this one compares 2000 and 16000 lines + # outright, both ways the text can arrive. + word_scaling_check("word-heavy file via -n FILE", [SHELL, "-n"]) + word_scaling_check("word-heavy file via source", [SHELL, "-c", ". "]) + # Re-sourcing an rc that installed aliases, functions and a DEBUG # trap must stay as cheap as the first load: hellish once disabled # the forkless $() path on the first alias and fired the DEBUG trap