From 3a3553cbb6745d56815efb09aef5fdf510c190aa Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 00:31:53 +0300 Subject: [PATCH 1/8] refactor: extract shared value descent into types/navigation.py --- docs/05-type-model.md | 8 ++++ fastraml/service/datahover.py | 90 ++++++++++++++--------------------- fastraml/types/navigation.py | 74 ++++++++++++++++++++++++++++ tests/unit/test_navigation.py | 20 ++++++++ 4 files changed, 138 insertions(+), 54 deletions(-) create mode 100644 fastraml/types/navigation.py create mode 100644 tests/unit/test_navigation.py diff --git a/docs/05-type-model.md b/docs/05-type-model.md index 1e0d692..98056cc 100644 --- a/docs/05-type-model.md +++ b/docs/05-type-model.md @@ -198,6 +198,14 @@ same selection, but preserves all possible structural alternatives when no discriminator selects one; it does not validate the whole value to guess a branch. +`types/navigation.py` exposes `branches(base, value)` and `children(base, +value)` over positioned `ValueNode` data. Branches follow aliases and recursion +markers and preserve every structural union candidate unless a discriminator +selects one. Children report the governing declaration, expected type, source +key span and field name or array index. Invalid values retain known field +types; ordinary additional fields have no invented declaration. The service +uses this API for typed-data tokens, independently of hover presentation. + RAML field names shared by fragment, endpoint, and type decoders are defined in `facet_names.py`; diagnostic fields and JSON Schema keywords are separate. diff --git a/fastraml/service/datahover.py b/fastraml/service/datahover.py index 036dccb..371c8e9 100644 --- a/fastraml/service/datahover.py +++ b/fastraml/service/datahover.py @@ -11,9 +11,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from fastraml.errors import RamlError from fastraml.types.base import Property -from fastraml.types.complex_ import ArrayShape, ObjectShape, RecursiveShape, UnionShape +from fastraml.types.navigation import children if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -21,6 +20,7 @@ from fastraml.datanode import DataNode, ValueNode from fastraml.positions import Position from fastraml.types.base import BaseShape + from fastraml.types.navigation import TypedChild @dataclass(frozen=True, slots=True, eq=False) @@ -97,36 +97,41 @@ def in_range(self, uri: str, span: Position) -> list[DataTarget]: def _targets(root: DataRoot) -> Iterator[DataTarget]: if root.data.key_location is not None and root.data.key_pos.is_known: yield DataTarget(root.data.key_location, root.data.key_pos, root.base, root.name, root.role, is_key=True) - pending: list[tuple[ValueNode, BaseShape, str, str, bool | None]] = [ - (root.data.value, root.base, root.name, root.role, None) - ] + value, base, name, role = root.data.value, root.base, root.name, root.role + required: bool | None = None + pending: list[tuple[Iterator[TypedChild], str]] = [] seen: set[tuple[ValueNode, BaseShape]] = set() - while pending: - value, base, name, role, required = pending.pop() - if (value, base) in seen: - continue - seen.add((value, base)) - uri = value.location or root.data.location - # Collections have child key/value spans. Indexing their whole extent - # would make whitespace and unknown fields inherit the parent hover. - # An encoded JSON scalar has no nested spans: its root alone is known. - if value.is_scalar or (value.position.is_known and not _has_child_spans(value)): - yield DataTarget(uri, value.position, base, name, role, required) - for branch in _branches(base, value): - shape = branch.shape - if isinstance(shape, ObjectShape) and value.mapping is not None: - for entry in value.mapping.entries: - prop = shape.property_for(entry.key) - if prop is None: - continue - presence = prop.required if isinstance(prop, Property) else None - child_role = 'object property' if isinstance(prop, Property) else 'pattern-matched property' - yield DataTarget(uri, entry.key_pos, prop.base, entry.key, child_role, presence, is_key=True) - pending.append((entry.value, prop.base, entry.key, child_role, presence)) - elif isinstance(shape, ArrayShape) and shape.items is not None and value.sequence is not None: - pending.extend( - (item.value, shape.items, '', 'array item', None) for item in reversed(value.sequence.items) - ) + while True: + if (value, base) not in seen: + seen.add((value, base)) + uri = value.location or root.data.location + # Collections have child key/value spans. Indexing their whole extent + # would make whitespace and unknown fields inherit the parent hover. + # An encoded JSON scalar has no nested spans: its root alone is known. + if value.is_scalar or (value.position.is_known and not _has_child_spans(value)): + yield DataTarget(uri, value.position, base, name, role, required) + if not value.is_scalar: + pending.append((children(base, value), uri)) + # Resume one sibling at a time; frames borrow the model's child iterator + # instead of buffering every typed child and copying it into a worklist. + while pending: + siblings, uri = pending[-1] + child = next(siblings, None) + if child is None: + pending.pop() + continue + prop = child.declaration + required = prop.required if isinstance(prop, Property) else None + if prop is None: + name, role = '', 'array item' + else: + name = str(child.name) + role = 'object property' if isinstance(prop, Property) else 'pattern-matched property' + yield DataTarget(uri, child.key_pos, child.base, name, role, required, is_key=True) + value, base = child.value, child.base + break + else: + return def _has_child_spans(value: ValueNode) -> bool: @@ -135,26 +140,3 @@ def _has_child_spans(value: ValueNode) -> bool: if value.sequence is not None: return any(item.value_pos.is_known for item in value.sequence.items) return False - - -def _branches(base: BaseShape, value: ValueNode) -> Iterator[BaseShape]: - pending = [base] - seen: set[BaseShape] = set() - while pending: - current = pending.pop() - if current in seen: - continue - seen.add(current) - shape = current.shape - if isinstance(shape, RecursiveShape): - pending.append(shape.head) - elif current.alias is not None: - pending.append(current.alias) - elif isinstance(shape, UnionShape): - try: - selected = shape.select(value.raw) - except RamlError: - continue # a discriminator that names no alternative selects none - pending.extend(reversed(shape.any_of or ()) if selected is None else (selected,)) - else: - yield current diff --git a/fastraml/types/navigation.py b/fastraml/types/navigation.py new file mode 100644 index 0000000..2448f74 --- /dev/null +++ b/fastraml/types/navigation.py @@ -0,0 +1,74 @@ +"""Typed-value descent without using validation to guess a union alternative. + +Matching uses the same property and discriminator operations as validation +(docs/05 § 4 and § 6). Callers decide how to present ambiguous candidates. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from fastraml.errors import RamlError +from fastraml.positions import UNKNOWN, Position +from fastraml.types.complex_ import ArrayShape, ObjectShape, RecursiveShape, UnionShape + +if TYPE_CHECKING: + from collections.abc import Iterator + + from fastraml.datanode import ValueNode + from fastraml.types.base import BaseShape, PatternProperty, Property + + +@dataclass(frozen=True, slots=True, eq=False) +class TypedChild: + name: str | int + value: ValueNode + base: BaseShape + declaration: Property | PatternProperty | None = None + key_pos: Position = UNKNOWN + + +def branches(base: BaseShape, value: ValueNode) -> Iterator[BaseShape]: + """All structural candidates, or none for an unknown discriminator tag.""" + pending = [iter((base,))] + seen: set[BaseShape] = set() + while pending: + current = next(pending[-1], None) + if current is None: + pending.pop() + continue + if current in seen: + continue + seen.add(current) + shape = current.shape + if isinstance(shape, RecursiveShape): + pending.append(iter((shape.head,))) + elif current.alias is not None: + pending.append(iter((current.alias,))) + elif isinstance(shape, UnionShape): + try: + selected = shape.select(value.raw) + except RamlError: + continue + pending.append(iter(shape.any_of or ()) if selected is None else iter((selected,))) + else: + yield current + + +def children(base: BaseShape, value: ValueNode) -> Iterator[TypedChild]: + """The declarations governing each immediate child, in branch/data order. + + Invalid values retain known field types; additional fields have none. Array + items are identified by integer indices, not invented property declarations. + """ + for branch in branches(base, value): + shape = branch.shape + if isinstance(shape, ObjectShape) and value.mapping is not None: + for entry in value.mapping.entries: + prop = shape.property_for(entry.key) + if prop is not None: + yield TypedChild(entry.key, entry.value, prop.base, prop, entry.key_pos) + elif isinstance(shape, ArrayShape) and shape.items is not None and value.sequence is not None: + for index, item in enumerate(value.sequence.items): + yield TypedChild(index, item.value, shape.items) diff --git a/tests/unit/test_navigation.py b/tests/unit/test_navigation.py new file mode 100644 index 0000000..eb090b4 --- /dev/null +++ b/tests/unit/test_navigation.py @@ -0,0 +1,20 @@ +"""Typed-value descent (docs/05 § 6).""" + +from __future__ import annotations + +from fastraml.parser.entry import ParseOptions +from fastraml.types.navigation import children + + +def test_typed_navigation_keeps_ambiguous_and_invalid_field_candidates(workspace): + text = ( + '#%RAML 1.0\ntitle: T\ntypes:\n' + ' A: {properties: {name: string}}\n' + ' B: {properties: {name: integer}}\n' + ' Both:\n type: A | B\n example: {name: false, extra: 1}\n' + ) + raml = workspace.document(text, ParseOptions(unwrap=True)) + base = raml.types_in(raml.location)['Both'] + found = list(children(base, base.example.data.value)) + assert [(each.name, each.base.type) for each in found] == [('name', 'string'), ('name', 'integer')] + assert all(each.key_pos.is_known for each in found) From 2873223ddc325076bf7a0cf5a323238b1031d6ed Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 01:25:24 +0300 Subject: [PATCH 2/8] fix: keep post-parse include reads out of the parse's text store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_include serves readers outside the parse, such as join comparing a target the parse never read (docs/20 § 2). It composes through _compose_include, which stored the decoded text in raml.source_texts whenever retain_text was on, contrary to its documented contract that such reads enter none of the parse's caches. A post-parse write could grow the read set of an already built snapshot. Add a store_text switch to _compose_include; load_include passes False. --- fastraml/parser/includes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fastraml/parser/includes.py b/fastraml/parser/includes.py index 1aeb2fa..037f039 100644 --- a/fastraml/parser/includes.py +++ b/fastraml/parser/includes.py @@ -285,15 +285,19 @@ def load_include(raml: Raml, node: Node, target: str, location: str) -> Node: through no cache and recording nothing: for a reader outside the parse, such as `join` comparing a target no applied template reached (docs/20 § 2). """ - return _compose_include(raml, node, _load(raml, node, target, location), target, location)[0] + return _compose_include(raml, node, _load(raml, node, target, location), target, location, store_text=False)[0] -def _compose_include(raml: Raml, node: Node, data: bytes, target: str, location: str) -> tuple[Node, str]: +def _compose_include( # noqa: PLR0913 - include coordinates plus text retention + raml: Raml, node: Node, data: bytes, target: str, location: str, *, store_text: bool = True +) -> tuple[Node, str]: """The include's content, and its RAML header line, `''` when it has none.""" try: text = decode_source(data) except UnicodeDecodeError as err: raise node_error('include is not UTF-8', location, node, info={'path': target}) from err + if store_text: + raml.store_source_text(target, text) ref = node.value if not _composes_as_yaml(ref): # Spec section Resolving Includes: any other file is included as a scalar. From 2e724b56e2b6f1cc00ed2f1696d6e0ad92f60391 Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 13:05:58 +0300 Subject: [PATCH 3/8] feat: record deprecated syntax spellings for the lint rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RAML 1.0 deprecates the schemas: table and the schema: facet. The decoders that accept those spellings now record each accepted use in Raml.syntax_aliases, keyed by the declaring entity's ID (docs/04 § 5, docs/05 § 3). The record holds the authored URI, the key's position and the spelling itself; it keeps no YAML node (docs/13 § 3). deprecated-schemas reads those records instead of walking every declaration's retained tree, so it is no longer a source-sensitive rule (docs/18 § 6). No rule in the default set now requires retained trees; the language service's snapshots therefore parse with text alone unless an opt-in style rule is enabled (docs/21 § 2). The retired docs/15 paragraph proposed exactly this recording. The three opt-in style rules that do read trees are named in docs/18. Bench (ab 2b6502e, 3 rounds): hover time +27.1% (noise 3.3%), peak +16.6%, kept +4.0%; inlays time +21.9% (noise 4.8%), peak +12.9%, kept +3.3%. The snapshots no longer retain parse trees, so the first hover or inlay per snapshot composes the queried file on demand, about a third of a parse of it; repeated queries are cached. Memory: large/service peak -20.5%, kept -24.4%; effective-types kept -27.5%. Accepted with a revisit note in docs/15 § 2. --- docs/04-fragments-and-namespaces.md | 6 ++++ docs/05-type-model.md | 6 ++++ docs/13-public-api.md | 6 ++++ docs/15-implementation-plan.md | 18 +++++----- docs/18-linting.md | 7 +++- docs/21-language-service.md | 3 +- fastraml/parser/fragments.py | 2 ++ fastraml/registry.py | 10 ++++++ fastraml/sourceinfo.py | 23 +++++++++++++ fastraml/types/shape.py | 2 ++ fastraml/views/lint/rules/schema.py | 49 +++++++++------------------- tests/unit/test_lint.py | 21 +++++++++++- tests/unit/test_service_workspace.py | 9 ++--- 13 files changed, 113 insertions(+), 49 deletions(-) create mode 100644 fastraml/sourceinfo.py diff --git a/docs/04-fragments-and-namespaces.md b/docs/04-fragments-and-namespaces.md index 795542e..ad36519 100644 --- a/docs/04-fragments-and-namespaces.md +++ b/docs/04-fragments-and-namespaces.md @@ -134,6 +134,12 @@ resolves every name through the includer and is located in the included file. ## 5. Fragment decoding +When a declaration table uses the accepted compatibility spelling `schemas:`, +the decoder records its name, authored URI and key position in +`Raml.syntax_aliases`, keyed by the fragment's entity ID. This is an authored +language fact, independent of source retention. The lint view decides whether +to report it and with what severity. + API decoding first collects global `mediaType`, `protocols`, and `securedBy`, then decodes remaining declarations in source order. A lone `mediaType` may be written as an annotated scalar, `{value: ..., (a): ...}`; its annotations diff --git a/docs/05-type-model.md b/docs/05-type-model.md index 98056cc..5f00f85 100644 --- a/docs/05-type-model.md +++ b/docs/05-type-model.md @@ -74,6 +74,12 @@ reference or inline declaration, not a bare sequence; use ## 3. Decoding and inference +An accepted `schema:` compatibility facet records its authored URI, position +and spelling in `Raml.syntax_aliases`, keyed by the shape's ID. It needs no +retained node and has no severity or diagnostic attached. A lint rule consumes +this fact rather than rediscovering it from every declaration's YAML tree. +Keys in examples, annotations or other opaque data do not declare such a facet. + `make_shape()` is the general declaration constructor. It records common facets on the base, keeps remaining YAML key/value pairs flat, determines or defers the kind, attaches it, builds the declaration facets (`properties`, `items`, diff --git a/docs/13-public-api.md b/docs/13-public-api.md index 4609c6d..a2dedbe 100644 --- a/docs/13-public-api.md +++ b/docs/13-public-api.md @@ -130,6 +130,12 @@ returns RAML text ([20](20-join.md) § 8). It is not part of `fastraml.__all__`. Pass `unwrap=True, validate=True` together unless you specifically need declared, unflattened types. Validation alone must clone and unwrap declarations privately. +Default lint no longer needs whole trees merely to detect `schema:` and +`schemas:`: the decoders record their accepted spelling and position in +`Raml.syntax_aliases` (docs/04 § 5, docs/05 § 3), independent of source +retention. A consumer needing only that compact fact should read it rather +than retaining the whole source tree. + A configuration file's `parser:` section is `ParserConfig`. `limits(options)` applies its `max_include_size`, `max_depth` and `regex_engine`; its `workspace_root` and `remote` are left to the host, which weighs them against diff --git a/docs/15-implementation-plan.md b/docs/15-implementation-plan.md index aeb3398..f6d38e6 100644 --- a/docs/15-implementation-plan.md +++ b/docs/15-implementation-plan.md @@ -59,6 +59,15 @@ the media-type fixes and the type-walk work landed together: endpoint build still holds; skip the swap where the node is retained anyway. - `validate`: time up to +4.9% on some configurations in a 3-round run; a 5-round rerun of `unwrap+validate` was within noise. +- `hover`/`unwrap` and `inlays`/`unwrap` (measured against `2b6502e` when the + service's snapshots stopped retaining parse trees): time +27.1% (noise + 3.3%) and +21.9% (noise 4.8%), peak +16.6% and +12.9%, kept +4.0% and + +3.3%. The cost is one composition of the queried file per snapshot, paid + by the first hover or inlay that needs its tree (docs/21 § 2); repeated + queries are cached, and the edit loop is within noise on time, peak + -20.5%, kept -24.4% (docs/21 § 4). Revisit: a clean file is composed + twice per snapshot, once for the hover index and once for the workspace's + source cache; sharing one tree between them removes the second. ## 3. Potential future work @@ -75,15 +84,6 @@ M4 are done, and completion (M5) comes next. Its design records are its parser prerequisite, a trustworthy `parse_lenient` model, is analysed in [research/partial-models.md](research/partial-models.md). -**Deprecated aliases as a model fact (proposed).** `deprecated-schemas` is on -by default and reads `source_info`, so every lint-enabled parse, the language -service's included, retains each declaration's YAML subtree: +7.66 MB on -`large`, against +0.56 MB for the text alone. It needs only where a `schema:` -or `schemas:` key was accepted, which the decoder knows. Recording those -(file, position, key) on the model would let the default rule set run on -`retain_text`; the three off-by-default rules that inspect how a declaration -was written keep `retain_source`. - **Sampled examples in the tree (undecided).** The viewer's request and response panel (parked on `feat/viewer-request-samples`) needs a working body for each payload, and only Python can produce a validated one. The candidate design: diff --git a/docs/18-linting.md b/docs/18-linting.md index 8a53af9..9b8c333 100644 --- a/docs/18-linting.md +++ b/docs/18-linting.md @@ -281,7 +281,12 @@ CLI builds the graph as part of each lint invocation. Lint requires an unwrapped model. Source-sensitive rules additionally require `ParseOptions(retain_source=True)`; `Linter.requires_source` says whether an -enabled rule is one. +enabled rule is one. Those are the opt-in style rules +`prefer-array-expression`, `avoid-explicit-inferred-type` and +`prefer-inline-alias`; no rule in the default set reads the trees. + +`deprecated-schemas` reads the decoder's `Raml.syntax_aliases` records and +works with any retention choice. It is not a source-sensitive rule. ## 7. What lint is not diff --git a/docs/21-language-service.md b/docs/21-language-service.md index 0fcb906..a5a183d 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -53,7 +53,8 @@ the TCK's 1011 files. **Snapshots.** A snapshot is one `parse_lenient` of one root, with `unwrap=True, validate=True, retain_text=True` and the configuration's `parser:` limits. It keeps the YAML trees too, `retain_source=True`, only when -an enabled lint rule reads them (`deprecated-schemas`, in the default set). +an enabled lint rule reads them; no rule in the default set does. Decoders +record the small syntax facts that `deprecated-schemas` reads (docs/18 § 6). It records the set of files it read: every retained text, every fragment, and every include it tried, found or not. It is built when a query first asks for it and kept until one of those files changes. A file that diff --git a/fastraml/parser/fragments.py b/fastraml/parser/fragments.py index 2deb538..70804ea 100644 --- a/fastraml/parser/fragments.py +++ b/fastraml/parser/fragments.py @@ -542,6 +542,8 @@ def _decode_declarations(self, key: Node, value: Node, declarations: _Declaratio raml = self._raml name = key.value if name in (FACET_TYPES, FACET_SCHEMAS): + if name == FACET_SCHEMAS: + raml.record_syntax_alias(self, key, self.location) unmarshal_types(raml, declarations.types(key, value), self.location, self.types) elif name == FACET_ANNOTATION_TYPES: unmarshal_types(raml, value, self.location, self.annotation_types, is_annotation=True) diff --git a/fastraml/registry.py b/fastraml/registry.py index 10bc476..d02bb72 100644 --- a/fastraml/registry.py +++ b/fastraml/registry.py @@ -23,6 +23,7 @@ from fastraml.domains import DomainLocation from fastraml.errors import Accumulator, RamlError from fastraml.loaders import SchemeLoader +from fastraml.sourceinfo import KeywordUse from fastraml.yamlnode import AUTHORED_NODES, DEFAULT_MAX_DEPTH, mark_subtree if TYPE_CHECKING: @@ -211,6 +212,7 @@ class Raml: 'include_refs', 'shapes', 'substitutions', + 'syntax_aliases', # --- work queues ----------------------------------------------------- '_discriminator_shapes', 'unresolved_shapes', @@ -297,6 +299,9 @@ def __init__( # noqa: PLR0913 - the parse's configuration, keyword-only, one fi #: Each scalar a template substitution produced, and the caller's #: values in it: where a name in it was written (docs/08 § 5.1). self.substitutions: Substitutions = {} + #: Accepted compatibility spellings by entity ID; only used spellings + #: allocate a record. The parser records syntax, a view judges it. + self.syntax_aliases: dict[int, list[KeywordUse]] = {} # A worklist, drained from the left in P7 while resolution appends to # the right; a deque keeps both ends O(1). @@ -634,6 +639,11 @@ def store_source_text(self, uri: str, text: str) -> None: if self.retain_text: self.source_texts[uri] = text + def record_syntax_alias(self, entity: Identified, key: Node, location: str) -> None: + """Where a decoder accepted a compatibility keyword spelling.""" + use = KeywordUse(self.location_of(key, location), key.position, key.value) + self.syntax_aliases.setdefault(entity.id, []).append(use) + def put_source_info(self, entity_id: int, key: Node | None, value: Node) -> None: """Index an entity's authored nodes when source retention is on.""" if self.source_info is not None: diff --git a/fastraml/sourceinfo.py b/fastraml/sourceinfo.py new file mode 100644 index 0000000..2e566be --- /dev/null +++ b/fastraml/sourceinfo.py @@ -0,0 +1,23 @@ +"""Facts a decoder records when it accepts a compatibility spelling. + +A `KeywordUse` names where a deprecated keyword was written. It is a model +fact the registry stores and a view may report on: not a diagnostic, and it +keeps no YAML node (docs/04 § 5, docs/05 § 3). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastraml.positions import Position + + +@dataclass(frozen=True, slots=True, eq=False) +class KeywordUse: + """An accepted compatibility spelling, not a lint finding or retained tree.""" + + location: str + position: Position + name: str diff --git a/fastraml/types/shape.py b/fastraml/types/shape.py index 962316f..1fc69aa 100644 --- a/fastraml/types/shape.py +++ b/fastraml/types/shape.py @@ -374,6 +374,8 @@ def _decode( # noqa: PLR0912 - one pass over the common-facet vocabulary (docs/ if type_node is not None: raise node_error('type and schema are mutually exclusive', location, key) type_node = value + if key.value == fn.FACET_SCHEMA: + raml.record_syntax_alias(base, key, location) case fn.FACET_DISPLAY_NAME: base.display_name = make_string_facet(raml, key, value, location) case fn.FACET_DESCRIPTION: diff --git a/fastraml/views/lint/rules/schema.py b/fastraml/views/lint/rules/schema.py index 873fd1e..25a6bd8 100644 --- a/fastraml/views/lint/rules/schema.py +++ b/fastraml/views/lint/rules/schema.py @@ -19,7 +19,6 @@ from fastraml.views.graph import is_declaration from fastraml.views.lint.engine import Category, Finding, RuleMeta, Severity from fastraml.views.lint.mediatypes import media_essence -from fastraml.yamlnode import NodeKind, pairs if TYPE_CHECKING: from collections.abc import Iterable @@ -45,8 +44,6 @@ class DeprecatedSchemas: """The deprecated `schemas:` alias for `types:`, and `schema:` for `type:`.""" - requires_source: ClassVar = True - meta: ClassVar = RuleMeta( id='deprecated-schemas', category=Category.SPEC, @@ -67,46 +64,32 @@ class DeprecatedSchemas: ) def type_(self, ctx: Context, iri: str, base: BaseShape, shape_kind: str) -> Iterable[Finding]: # noqa: ARG002 - authored = None if ctx.raml.source_info is None else ctx.raml.source_info.get(base.id) - if authored is None or authored[1].kind is not NodeKind.MAPPING: - return () - # Keys only, sliced rather than paired: this runs once per type node, - # and most declarations have nothing to report. return [ ctx.at( self.meta, 'schema is deprecated; use type', - location=base.location, - position=key.position, + location=use.location, + position=use.position, iri=iri, field='schema', ) - for key in authored[1].content[::2] - if key.value == 'schema' + for use in ctx.raml.syntax_aliases.get(base.id, ()) + if use.name == 'schema' ] def unit(self, ctx: Context, iri: str, fragment: Fragment) -> Iterable[Finding]: - root = ctx.raml.source_node(fragment.location) - if root is None: - # A `type: !include schema.json` is represented by a data-type - # fragment and a graph unit, but JSON input never had a RAML root - # node to retain. Only composed RAML sources can spell `schemas:`. - return () - if root.kind is not NodeKind.MAPPING: - return () - for key, _ in pairs(root): - if key.value == 'schemas': - return ( - ctx.at( - self.meta, - 'schemas is deprecated; use types', - location=fragment.location, - position=key.position, - iri=iri, - field='schemas', - ), - ) - return () + return [ + ctx.at( + self.meta, + 'schemas is deprecated; use types', + location=use.location, + position=use.position, + iri=iri, + field='schemas', + ) + for use in ctx.raml.syntax_aliases.get(fragment.id, ()) + if use.name == 'schemas' + ] class JsonRefSiblings: diff --git a/tests/unit/test_lint.py b/tests/unit/test_lint.py index 6a2131d..951e287 100644 --- a/tests/unit/test_lint.py +++ b/tests/unit/test_lint.py @@ -795,12 +795,13 @@ def test_suppression_uses_an_included_files_location(self, workspace): def run_rule(rule_id: str, source: str, tmp_path, **options): + retain_source = options.pop('retain_source', True) config = Config(extends=(), rules=(RuleSetting(id=rule_id),)) raml = parse_from_string( source, file_name='api.raml', base_dir=tmp_path, - options=ParseOptions(unwrap=True, validate=False, retain_source=True, **options), + options=ParseOptions(unwrap=True, validate=False, retain_source=retain_source, **options), ) return Linter(builtin_registry(), config).run(raml) @@ -1236,6 +1237,24 @@ def test_deprecated_schemas_ignores_a_property_named_schema(self, tmp_path): source = '#%RAML 1.0\ntitle: t\ntypes:\n A:\n properties:\n schema: string\n' assert not run_rule('deprecated-schemas', source, tmp_path) + def test_deprecated_schemas_reports_without_retained_source(self, tmp_path): + # The decoder records each accepted spelling (docs/04 § 5, docs/05 § 3), + # so the rule runs with any retention choice (docs/18 § 6). + table = run_rule( + 'deprecated-schemas', + '#%RAML 1.0\ntitle: t\nschemas:\n U: string\n', + tmp_path, + retain_source=False, + ) + assert [(finding.position.line, finding.info) for finding in table] == [(3, {'field': 'schemas'})] + facet = run_rule( + 'deprecated-schemas', + '#%RAML 1.0\ntitle: t\ntypes:\n A:\n schema: string\n', + tmp_path, + retain_source=False, + ) + assert [(finding.position.line, finding.info) for finding in facet] == [(5, {'field': 'schema'})] + @pytest.mark.parametrize( ('resources', 'expected'), [ diff --git a/tests/unit/test_service_workspace.py b/tests/unit/test_service_workspace.py index 742c305..000f17d 100644 --- a/tests/unit/test_service_workspace.py +++ b/tests/unit/test_service_workspace.py @@ -301,11 +301,12 @@ def test_collect_frees_the_dropped_snapshots(self, tmp_path): def test_the_yaml_trees_are_kept_only_for_a_lint_rule_that_reads_them(self, tmp_path): files = {'api.raml': API} reading, folder = _workspace(tmp_path, files) - assert reading.snapshot(f'{folder}/api.raml').raml.retain_source - config = FastRamlConfig(lint={'rules': [{'id': 'deprecated-schemas', 'disabled': True}]}) - plain = Workspace([folder], config=config) - raml = plain.snapshot(f'{folder}/api.raml').raml + raml = reading.snapshot(f'{folder}/api.raml').raml assert (raml.retain_source, raml.retain_text) == (False, True) + config = FastRamlConfig(lint={'rules': [{'id': 'prefer-array-expression'}]}) + syntax = Workspace([folder], config=config) + raml = syntax.snapshot(f'{folder}/api.raml').raml + assert (raml.retain_source, raml.retain_text) == (True, True) def test_the_first_snapshot_serving_a_file_parses_one_root(self, tmp_path, monkeypatch): # After an edit to a library, the outline read one snapshot but brought From 885a16fbcce7871187cbea6053c6d78b5149fb22 Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 13:25:55 +0300 Subject: [PATCH 4/8] feat: enumerate resolver-visible names for completion --- docs/04-fragments-and-namespaces.md | 12 +++++++++ docs/21-language-service.md | 1 + fastraml/parser/fragments.py | 38 ++++++++++++++++++++++++++++- fastraml/parser/includes.py | 7 +++++- fastraml/parser/references.py | 29 +++++++++++++++++++++- fastraml/service/queries.py | 14 +++++++++++ tests/unit/test_service_queries.py | 26 ++++++++++++++++++++ 7 files changed, 124 insertions(+), 3 deletions(-) diff --git a/docs/04-fragments-and-namespaces.md b/docs/04-fragments-and-namespaces.md index ad36519..40f1741 100644 --- a/docs/04-fragments-and-namespaces.md +++ b/docs/04-fragments-and-namespaces.md @@ -45,6 +45,7 @@ class ReferenceResolver(Fragment, Protocol): def trait_definition(self, name: str) -> TraitDefinition: ... def security_scheme_definition(self, name: str) -> SecuritySchemeDefinition: ... def library_link(self, prefix: str) -> LibraryLink | None: ... + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: ... ``` `ReferenceResolver` includes the `SecuritySchemeResolver` capability. Every @@ -55,6 +56,17 @@ declarations, as does an Overlay or Extension in its target tree's declarations implementation over two hooks: the local table and the `uses:` map. Resolver capabilities are checked structurally. +`visible_names(kind)` enumerates names for one declaration table (`types`, +`annotationTypes`, `traits`, `resourceTypes`, or `securitySchemes`). It yields +local declarations first, then each directly imported library's declarations +with that import's prefix. Local dotted names take precedence. Every qualified +candidate must round-trip through ordinary resolution to the same entity; +unresolved imports and inaccessible dotted suffixes contribute no names. +It never follows a library's own imports. Annotation-type enumeration includes +the ordinary-type fallback, with annotation types taking precedence. +Extension resolvers enumerate only the declarations and imports visible at +their chain position. Literal included content delegates to its host resolver. + ## 3. Name resolution `resolve_reference()` resolves declarations in a fragment with a local table. diff --git a/docs/21-language-service.md b/docs/21-language-service.md index a5a183d..3c34b8d 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -124,6 +124,7 @@ composed YAML and the parser's structural grammar. | `workspace_symbols` | the declarations of every snapshot, matched case-insensitively, once each | | `links` | each `Link` occurrence in the file, with the file it resolved to | | `folding_ranges`, `selection_ranges` | the buffer's composed `Node` tree alone | +| `visible_names` | the fragment resolver's local and directly imported declaration candidates (docs/04 § 2) | | `type_at`, `supertypes`, `subtypes` | a type's `inherits` and `alias`, and the declarations naming it | | `diagnostics` | `RamlError.chains()` and the lint findings | | `tree` | `build_tree` (`docs/16` § 6) as JSON text, only on an unwrapped model | diff --git a/fastraml/parser/fragments.py b/fastraml/parser/fragments.py index 70804ea..389dad1 100644 --- a/fastraml/parser/fragments.py +++ b/fastraml/parser/fragments.py @@ -66,7 +66,7 @@ resolve_ref_uri, strip_uri_suffix, ) -from fastraml.parser.references import resolve_library_reference, resolve_reference +from fastraml.parser.references import resolve_library_reference, resolve_reference, visible_references from fastraml.parser.resourcetypes import ResourceTypeDefinition, make_resource_type_definition from fastraml.parser.security import SecuritySchemeDefinition, make_security_scheme_definition from fastraml.parser.traits import TraitDefinition, make_trait_definition @@ -220,6 +220,8 @@ class ReferenceResolver(Fragment, SecuritySchemeResolver, Protocol): and therefore all of them may resolve a qualified name. """ + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: ... + def reference_type(self, name: str) -> BaseShape: ... def reference_annotation_type(self, name: str) -> BaseShape: ... @@ -394,6 +396,9 @@ class _UsesOnlyFragment(_BaseFragment): __slots__ = () + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: + return _visible_declarations(None, None, self.uses, kind) + def reference_type(self, name: str) -> BaseShape: return resolve_library_reference(self.uses, name, _pick_type) @@ -445,6 +450,32 @@ def _pick_security_scheme(library: Library, name: str) -> SecuritySchemeDefiniti } +def _visible_table(library: Library, kind: str) -> Mapping[str, Declaration]: + table: Mapping[str, Declaration] = getattr(library, _DECLARATION_TABLES[kind]) + return table + + +def _visible_declarations( + local: Mapping[str, Declaration] | None, + types: Mapping[str, Declaration] | None, + uses: Mapping[str, LibraryLink], + kind: str, +) -> Iterator[tuple[str, Declaration]]: + primary = visible_references(local, uses, lambda library: _visible_table(library, kind)) + if kind != FACET_ANNOTATION_TYPES: + yield from primary + return + seen: set[str] = set() + for name, entity in primary: + seen.add(name) + yield name, entity + # Annotation lookup tries the whole annotation namespace before falling + # back to the type namespace, including before a local dotted data type. + for name, entity in visible_references(types, uses, lambda library: _visible_table(library, FACET_TYPES)): + if name not in seen: + yield name, entity + + class _NameResolver: """The five name resolvers, written once over two hooks. @@ -469,6 +500,11 @@ def _resolve[T](self, kind: str, name: str, pick: Callable[[Library, str], T | N def library_link(self, prefix: str) -> LibraryLink | None: return self._libraries().get(prefix) + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: + local: Mapping[str, Declaration] | None = self._declared(kind) + types = self._declared(FACET_TYPES) if kind == FACET_ANNOTATION_TYPES else None + return _visible_declarations(local, types, self._libraries(), kind) + def reference_type(self, name: str) -> BaseShape: return self._resolve(FACET_TYPES, name, _pick_type) diff --git a/fastraml/parser/includes.py b/fastraml/parser/includes.py index 037f039..830ed80 100644 --- a/fastraml/parser/includes.py +++ b/fastraml/parser/includes.py @@ -34,7 +34,9 @@ ) if TYPE_CHECKING: - from fastraml.parser.fragments import LibraryLink, ReferenceResolver + from collections.abc import Iterator + + from fastraml.parser.fragments import Declaration, LibraryLink, ReferenceResolver from fastraml.parser.resourcetypes import ResourceTypeDefinition from fastraml.parser.security import SecuritySchemeDefinition from fastraml.parser.traits import TraitDefinition @@ -389,6 +391,9 @@ def security_scheme_definition(self, name: str) -> SecuritySchemeDefinition: def library_link(self, prefix: str) -> LibraryLink | None: return self.host.library_link(prefix) + def visible_names(self, kind: str) -> Iterator[tuple[str, Declaration]]: + return self.host.visible_names(kind) + def content_anchor(anchor: ReferenceResolver | None, location: str) -> ReferenceResolver | None: """`anchor`, or an `IncludedContent` of it when what is decoded at diff --git a/fastraml/parser/references.py b/fastraml/parser/references.py index 009d561..3da0250 100644 --- a/fastraml/parser/references.py +++ b/fastraml/parser/references.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Iterator, Mapping from fastraml.parser.fragments import Library, LibraryLink @@ -22,6 +22,7 @@ 'cut_last', 'resolve_library_reference', 'resolve_reference', + 'visible_references', ] @@ -109,6 +110,32 @@ def resolve_library_reference[T]( return _pick_from_library(uses, prefix, suffix, pick) +def visible_references[T]( + local: Mapping[str, T] | None, + uses: Mapping[str, LibraryLink], + table: Callable[[Library], Mapping[str, T]], +) -> Iterator[tuple[str, T]]: + """Enumerate names that resolve in this namespace, with local precedence. + + Only one library hop, including dotted import prefixes. Check the round + trip rather than inventing a second interpretation of dotted names. + """ + yield from (local or {}).items() + for prefix, link in uses.items(): + if link.link is None: + continue + for name, entity in table(link.link).items(): + qualified = f'{prefix}.{name}' + if local is not None and qualified in local: + continue + try: + resolved = resolve_reference(local, uses, qualified, lambda library, key: table(library).get(key)) + except UnresolvedReferenceError: + continue + if resolved is entity: + yield qualified, entity + + def _pick_from_library[T]( uses: Mapping[str, LibraryLink] | None, prefix: str, diff --git a/fastraml/service/queries.py b/fastraml/service/queries.py index a013f76..b7b91d9 100644 --- a/fastraml/service/queries.py +++ b/fastraml/service/queries.py @@ -58,6 +58,7 @@ 'symbol', 'tree', 'type_at', + 'visible_names', 'workspace_symbols', ] @@ -374,6 +375,19 @@ def workspace_symbols(snapshots: Iterable[Snapshot], query: str) -> list[Symbol] return found +def visible_names(snapshot: Snapshot, uri: str, kind: str) -> list[Symbol]: + """Completion candidates from a fragment's own resolver, never all declarations.""" + raml = snapshot.raml + anchor = None if raml is None else raml.resolver_at(uri) + if anchor is None: + return [] + return [ + each + for name, entity in anchor.visible_names(kind) + if (each := symbol(name, DECLARATION_KINDS[kind], entity)) is not None + ] + + # -- links, folding and selection ------------------------------------------------------- diff --git a/tests/unit/test_service_queries.py b/tests/unit/test_service_queries.py index 2a32515..11e0bd8 100644 --- a/tests/unit/test_service_queries.py +++ b/tests/unit/test_service_queries.py @@ -137,6 +137,32 @@ def test_nothing_is_under_a_cursor_on_no_name(self, parsed): snapshot, folder = parsed assert queries.definition(snapshot, f'{folder}/api.raml', *_where(API, 'title')) == [] + def test_visible_names_obey_local_precedence_and_one_library_hop(self, memory_workspace): + api = '#%RAML 1.0\ntitle: T\nuses:\n lib: lib.raml\ntypes:\n lib.Person: string\n' + library = '#%RAML 1.0 Library\nuses:\n nested: other.raml\ntypes:\n Person: string\n Other: integer\n' + workspace, folder = _buffered( + memory_workspace, + { + 'api.raml': api, + 'lib.raml': library, + 'other.raml': '#%RAML 1.0 Library\ntypes:\n Hidden: boolean\n', + }, + ) + uri = f'{folder}/api.raml' + found = queries.visible_names(workspace.snapshot(uri), uri, 'types') + assert [(each.name, each.uri.rsplit('/', 1)[-1]) for each in found] == [ + ('lib.Person', 'api.raml'), + ('lib.Other', 'lib.raml'), + ] + + def test_extension_visible_names_exclude_declarations_from_later_chain_positions(self, memory_workspace): + api = '#%RAML 1.0\ntitle: T\ntypes:\n A: string\n' + extension = '#%RAML 1.0 Extension\nextends: api.raml\ntypes:\n B: string\n' + workspace, folder = _buffered(memory_workspace, {'api.raml': api, 'ext.raml': extension}) + snapshot = workspace.snapshot(f'{folder}/ext.raml') + assert [each.name for each in queries.visible_names(snapshot, f'{folder}/api.raml', 'types')] == ['A'] + assert [each.name for each in queries.visible_names(snapshot, f'{folder}/ext.raml', 'types')] == ['A', 'B'] + class TestHover: def test_a_type_describes_its_identity_without_an_effective_dump(self, parsed): From efd96304daa0f75c19fb683ad822a7af2e1fc78b Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 13:47:56 +0300 Subject: [PATCH 5/8] perf: cache composed source trees in the workspace for folding and selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Folding and selection composed the buffer's whole document on every request (docs/21 § 4). The workspace now composes a file's current text once and holds the tree, invalidated with the buffer on change, close or a disk change; Workspace.source answers both request kinds. The text-based folding_ranges and selection_ranges keep their one-shot, per-call composition. The source-structure reach test also counts compose calls: each measured pass composes the text once, shared by its fifteen requests. Bench (ab 2b6502e, 3 rounds): no existing workload moves beyond its noise; the hover and inlays first-use deltas belong to the preceding commit and are documented in docs/15 § 2. New source-structure workload, linearity over 3 repeats: time 1.004, peak 0.998, retained 1.000. See docs/reports/2026-10-10/service-source-simple-rework.md. --- bench/__main__.py | 36 +++++++++++++++++++++++++ docs/12-performance.md | 17 +++++++++++- docs/21-language-service.md | 6 ++--- fastraml/service/lsp.py | 10 ++++--- fastraml/service/queries.py | 30 +++++++++++++++++---- fastraml/service/workspace.py | 23 +++++++++++++++- tests/bench/test_corpus.py | 40 ++++++++++++++++++++++++++++ tests/unit/test_service_workspace.py | 33 +++++++++++++++++++++++ 8 files changed, 181 insertions(+), 14 deletions(-) diff --git a/bench/__main__.py b/bench/__main__.py index 7fa56dd..d56942b 100644 --- a/bench/__main__.py +++ b/bench/__main__.py @@ -124,6 +124,7 @@ def _at(base: int, scale: float) -> int: Bench('hover', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), Bench('effective-types', lambda root, scale: corpus.write_hover(root, family_count=_at(300, scale))), Bench('inlays', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), + Bench('source-structure', lambda root, scale: corpus.write_hover(root, family_count=_at(400, scale))), ) _BY_NAME = {bench.name: bench for bench in BENCHES} @@ -159,6 +160,7 @@ def run_one(bench: str, config: str, entry: Path, repeat: int) -> Measurement: 'hover', 'effective-types', 'inlays', + 'source-structure', }: return _measure_view(bench, entry, repeat) if config == 'service': @@ -202,6 +204,7 @@ def _measure_view(bench: str, entry: Path, repeat: int) -> Measurement: 'hover': _measure_hover, 'effective-types': _measure_effective_types, 'inlays': _measure_inlays, + 'source-structure': _measure_source_structure, }.get(bench) if service_workload is not None: return service_workload(entry, repeat) @@ -363,6 +366,38 @@ def hints() -> object: return measure('inlays', 'unwrap', hints, repeat=repeat) +def _measure_source_structure(entry: Path, repeat: int) -> Measurement: + """Folding and selection requests served from the workspace's source cache.""" + from fastraml.gctuning import tuned_gc # noqa: PLC0415 - feature workload only + from fastraml.service import queries # noqa: PLC0415 - feature workload only + from fastraml.service.workspace import Workspace # noqa: PLC0415 - feature workload only + from fastraml.uris import path_to_file_uri # noqa: PLC0415 - feature workload only + + text = entry.read_text(encoding='utf-8') + uri, folder = path_to_file_uri(entry), path_to_file_uri(entry.parent) + lines = text.splitlines() + probes = 12 + step = max(1, len(lines) // probes) + positions = [ + (number, len(raw) - len(raw.lstrip()) + 1) + for number, raw in enumerate(lines[2:], start=3) + if ':' in raw and not raw.lstrip().startswith('#') + ][::step][:probes] + + def structure() -> object: + workspace = Workspace([folder]) + workspace.open(uri, text, 1) + folded = [queries.folding_ranges_of(workspace.source(uri)) for _ in range(3)] + selected = [queries.selection_ranges_of(workspace.source(uri), line, column) for line, column in positions] + if len(positions) < probes or not folded[0] or any(not spans for spans in selected): + message = 'source-structure workload no longer reaches folding and selection' + raise RuntimeError(message) + return workspace.source(uri), folded, selected + + with tuned_gc(): + return measure('source-structure', 'unwrap', structure, repeat=repeat) + + def _measure_edit(bench: str, entry: Path, repeat: int) -> Measurement: """One edit to the root's buffer, and what the editor then asks for first.""" from itertools import count # noqa: PLC0415 - as above @@ -531,6 +566,7 @@ def compare(results: Sequence[Measurement], tolerance: float) -> int: 'hover': 'unwrap', 'effective-types': 'unwrap', 'inlays': 'unwrap', + 'source-structure': 'unwrap', } diff --git a/docs/12-performance.md b/docs/12-performance.md index ea01d61..1a55f55 100644 --- a/docs/12-performance.md +++ b/docs/12-performance.md @@ -120,9 +120,10 @@ must receive a parser diagnostic rather than `RecursionError`. | `hover` | a cold language-service snapshot, fourteen authoring hovers per type family and custom-facet definition queries: inherited summaries, facet explanations, built-ins, custom-facet declarations and supplied keys, nested custom-facet and annotation keys and values, property presence, methods and response statuses ([21](21-language-service.md) § 4.2) | | `effective-types` | a cold service snapshot, code-lens enumeration and full-depth RAML rendering for every named type and annotation type, including nested arrays, unions, scalar item constraints and recursive references ([21](21-language-service.md) § 4.3) | | `inlays` | a cold snapshot, inferred declaration types, expected types at supplied custom-facet/annotation roots and nested data keys, and compact inherited constraints anchored to type references ([21](21-language-service.md) § 4.4) | +| `source-structure` | three folding requests and twelve selection requests at scattered keys, served from the workspace's composed source tree, composed once per current text ([21](21-language-service.md) § 4) | The six general workloads are `small`, `large`, `endpoints`, `extensions`, -`validate` and `jsonschema`. The other twenty-four are feature workloads: +`validate` and `jsonschema`. The other twenty-five are feature workloads: each exists because no general workload runs the code it covers. Their reach tests (`tests/bench/test_corpus.py`) count calls or check bound results, and fail if a corpus stops reaching that code at every size it covers. @@ -155,6 +156,9 @@ The small corpus-validity tests run in the ordinary test suite. For `hover`, `unwrap` builds a service snapshot with unwrap and validation, then queries every generated hover site. It measures the cold indices and their reuse; probe selection and corpus generation are outside the measured region. +For `source-structure`, `unwrap` opens the corpus as a buffer and serves the +folding and selection requests from the workspace's composed source tree; +probe selection and corpus generation are outside the measured region. ```bash python -m bench run @@ -206,6 +210,17 @@ a change to a hot path, or to any code a claim is made about: `python -m bench linearity --bench NAME` for time and memory instead. Include the workload, both deltas, and the noise in the commit message. + +A service workload's measured region is a cold snapshot: the parse, plus the +first query of each kind, then reuse of the built indices. Read its delta in +two parts. The parse side is where the base revision built and retained every +file's tree and the comparison does not. The first-use side is where a query +composes on demand the tree the base kept for free, one composition per file +per snapshot, about a third of a parse of the file; repeated queries of the +same kind are cached and show no delta. When a change moves composition from +parse time to first use, name which side the delta is on before attributing +it to the change's hot path (docs/21 § 2, docs/21 § 4). + `bench compare` against the committed baseline is only a coarse check for large regressions: the baseline and the comparison run at different times, and the machine drifts in between. Profile before optimizing: use `cProfile` for diff --git a/docs/21-language-service.md b/docs/21-language-service.md index 3c34b8d..cd45cb6 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -20,7 +20,7 @@ and several views, and only `cli/` imports it (`docs/02` § 2; | Module | Holds | |---|---| -| `service/workspace.py` | buffers, the overlay loader, roots, snapshots, staleness | +| `service/workspace.py` | buffers, the overlay loader, roots, snapshots, staleness, composed source trees | | `service/text.py` | converting a column between fastRAML and a protocol | | `service/queries.py` | the queries, in fastRAML positions | | `service/outline.py` | the outline, over the authorship view (`docs/16` § 10) | @@ -123,7 +123,7 @@ composed YAML and the parser's structural grammar. | `document_symbols` | the fragment's metadata, declaration tables, documentation and resources, grouped by section, from `key_pos` and `value_pos`; `type_expr`, else `type_name`, for a type's detail | | `workspace_symbols` | the declarations of every snapshot, matched case-insensitively, once each | | `links` | each `Link` occurrence in the file, with the file it resolved to | -| `folding_ranges`, `selection_ranges` | the buffer's composed `Node` tree alone | +| `folding_ranges`, `selection_ranges` | the workspace's composed source tree for the file's current text, composed once per text | | `visible_names` | the fragment resolver's local and directly imported declaration candidates (docs/04 § 2) | | `type_at`, `supertypes`, `subtypes` | a type's `inherits` and `alias`, and the declarations naming it | | `diagnostics` | `RamlError.chains()` and the lint findings | @@ -428,7 +428,7 @@ current only those it reads: | definition, highlight, hover, type hierarchy, supertypes | the first snapshot that answers | | references, subtypes | every snapshot, answers deduplicated | | workspace symbols | every root | -| folding, selection | the buffer's text alone | +| folding, selection | the workspace's composed source tree for the buffer's current text, composed once per text (§ 4) | A file that roots bind differently (a master an Overlay merges into, a template applied with different arguments) answers from the first. References diff --git a/fastraml/service/lsp.py b/fastraml/service/lsp.py index c6db186..521d81d 100644 --- a/fastraml/service/lsp.py +++ b/fastraml/service/lsp.py @@ -497,19 +497,21 @@ def links(params: types.DocumentLinkParams) -> list[types.DocumentLink] | None: def folding(params: types.FoldingRangeParams) -> list[types.FoldingRange] | None: if (uri := _file(params.text_document.uri)) is None: return None - text = self.service.text(uri) or '' - return [types.FoldingRange(start - 1, end - 1) for start, end in queries.folding_ranges(text, uri)] + return [ + types.FoldingRange(start - 1, end - 1) + for start, end in queries.folding_ranges_of(self.service.source(uri)) + ] @feature(types.TEXT_DOCUMENT_SELECTION_RANGE) def selection(params: types.SelectionRangeParams) -> list[types.SelectionRange] | None: if (at := self._in(params.text_document.uri)) is None: return None uri, positions = at - text = self.service.text(uri) or '' + root = self.service.source(uri) found: list[types.SelectionRange] = [] for position in params.positions: parent: types.SelectionRange | None = None - for span in reversed(queries.selection_ranges(text, uri, *positions.to_server(uri, position))): + for span in reversed(queries.selection_ranges_of(root, *positions.to_server(uri, position))): parent = types.SelectionRange(positions.range(uri, span), parent) found.append(parent or types.SelectionRange(types.Range(position, position))) return found diff --git a/fastraml/service/queries.py b/fastraml/service/queries.py index b7b91d9..6dfb1a4 100644 --- a/fastraml/service/queries.py +++ b/fastraml/service/queries.py @@ -47,11 +47,13 @@ 'detail_line', 'diagnostics', 'folding_ranges', + 'folding_ranges_of', 'highlights', 'hover', 'links', 'references', 'selection_ranges', + 'selection_ranges_of', 'subtypes', 'supertypes', 'suppression', @@ -409,8 +411,27 @@ def links(snapshot: Snapshot, uri: str) -> list[Site]: def folding_ranges(text: str, uri: str) -> list[tuple[int, int]]: """Each mapping and sequence value spanning lines, from its key's line to its last leaf's, 1-based, from the text alone. + + The composition happens per call; repeated requests for one file should + use `Workspace.source`. + """ + return folding_ranges_of(_compose(text, uri)) + + +def selection_ranges(text: str, uri: str, line: int, column: int) -> list[Position]: + """The spans holding `line:column`, innermost first: a token, its pair, + the pair's mapping, and outward. + + The composition happens per call; repeated requests for one file should + use `Workspace.source`. + """ + return selection_ranges_of(_compose(text, uri), line, column) + + +def folding_ranges_of(root: Node | None) -> list[tuple[int, int]]: + """Each mapping and sequence value of `root` spanning lines, from its key's + line to its last leaf's, 1-based, from the tree alone. """ - root = _compose(text, uri) if root is None: return [] found: set[tuple[int, int]] = set() @@ -421,11 +442,10 @@ def folding_ranges(text: str, uri: str) -> list[tuple[int, int]]: return sorted(found) -def selection_ranges(text: str, uri: str, line: int, column: int) -> list[Position]: - """The spans holding `line:column`, innermost first: a token, its pair, - the pair's mapping, and outward. +def selection_ranges_of(root: Node | None, line: int, column: int) -> list[Position]: + """The spans of `root` holding `line:column`, innermost first: a token, its + pair, the pair's mapping, and outward, from the tree alone. """ - root = _compose(text, uri) if root is None: return [] found: list[Position] = [] diff --git a/fastraml/service/workspace.py b/fastraml/service/workspace.py index 7094749..3b3b9d2 100644 --- a/fastraml/service/workspace.py +++ b/fastraml/service/workspace.py @@ -35,7 +35,7 @@ from fastraml.uris import file_uri_to_path, path_to_file_uri, relative_to from fastraml.views.lint import configured_linter from fastraml.views.occurrences import build_occurrences -from fastraml.yamlnode import read_head +from fastraml.yamlnode import compose, read_head if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping, Sequence @@ -44,6 +44,7 @@ from fastraml.registry import Raml from fastraml.views.lint import Finding, Linter from fastraml.views.occurrences import Occurrences + from fastraml.yamlnode import Node __all__ = ['Buffer', 'Snapshot', 'Workspace', 'canonical'] @@ -180,6 +181,9 @@ def __init__( self.buffers: dict[str, Buffer] = {} self._roots: list[str] | None = None self._snapshots: dict[str, Snapshot] = {} + #: `uri` -> (the text it was composed from, the composed tree), one per + #: current text (docs/21 § 4). + self._sources: dict[str, tuple[str, Node]] = {} #: What each dropped snapshot read: the order `serving` tries roots in. self._last_read: dict[str, frozenset[str]] = {} #: Whether a snapshot was dropped since the last collection. @@ -232,6 +236,22 @@ def lines(self, uri: str) -> Lines: buffer = self.buffers.get(canonical(uri)) return buffer.lines if buffer is not None else Lines(self.text(uri) or '') + def source(self, uri: str) -> Node | None: + """The composed tree of `uri`'s current text, composed once per text (docs/21 § 4).""" + uri = canonical(uri) + text = self.text(uri) + if text is None: + return None + cached = self._sources.get(uri) + if cached is not None and cached[0] is text: + return cached[1] + try: + root = compose(text, uri=uri) + except RamlError: + return None + self._sources[uri] = (text, root) + return root + def _put(self, uri: str, buffer: Buffer) -> None: """Hold `buffer`, and drop what read other text for `uri`. @@ -264,6 +284,7 @@ def _touched(self, uri: str, *, appeared: bool) -> None: A file that appeared may be one a failed include was looking for, so a snapshot that ended in an error is dropped then too. """ + self._sources.pop(uri, None) for root, snapshot in list(self._snapshots.items()): if uri in snapshot.read or (appeared and snapshot.error is not None): del self._snapshots[root] diff --git a/tests/bench/test_corpus.py b/tests/bench/test_corpus.py index ff74167..2dc6bec 100644 --- a/tests/bench/test_corpus.py +++ b/tests/bench/test_corpus.py @@ -52,6 +52,7 @@ 'hover': lambda root: corpus.write_hover(root, family_count=3), 'effective-types': lambda root: corpus.write_hover(root, family_count=3), 'inlays': lambda root: corpus.write_hover(root, family_count=3), + 'source-structure': lambda root: corpus.write_hover(root, family_count=3), } @@ -126,6 +127,45 @@ def parse(entry, options): run_one('lenient-recovery', 'unwrap+lint', entry, repeat=1) assert len(results) == 4, 'timing and allocation both reach recovery and lint' + @pytest.mark.parametrize('count', [2, 4]) + def test_source_structure_reaches_folding_and_selection_on_the_cached_tree(self, tmp_path, monkeypatch, count): + from bench.__main__ import run_one + from fastraml.service import queries + from fastraml.service import workspace as workspace_module + + folded = [] + selected = [] + compositions = 0 + original_folding = queries.folding_ranges_of + original_selection = queries.selection_ranges_of + original_compose = workspace_module.compose + + def folding(root): + result = original_folding(root) + assert result + folded.append(len(result)) + return result + + def selection(root, line, column): + result = original_selection(root, line, column) + assert len(result) > 1, 'a position inside a nested structure spans at least root, pair and token' + selected.append(len(result)) + return result + + def compose(text, **kwargs): + nonlocal compositions + compositions += 1 + return original_compose(text, **kwargs) + + monkeypatch.setattr(queries, 'folding_ranges_of', folding) + monkeypatch.setattr(queries, 'selection_ranges_of', selection) + monkeypatch.setattr(workspace_module, 'compose', compose) + entry = corpus.write_hover(tmp_path, family_count=count) + run_one('source-structure', 'unwrap', entry, repeat=1) + assert len(folded) == 2 * 3, 'timing and allocation both reach folding' + assert len(selected) == 2 * 12, 'timing and allocation both reach selection' + assert compositions == 2, 'each measured pass composes the text once, shared by fifteen requests' + @pytest.mark.parametrize('count', [2, 4]) def test_inlays_reaches_inferred_types_data_types_and_inherited_facets(self, tmp_path, monkeypatch, count): from bench.__main__ import run_one diff --git a/tests/unit/test_service_workspace.py b/tests/unit/test_service_workspace.py index 000f17d..3fdcef6 100644 --- a/tests/unit/test_service_workspace.py +++ b/tests/unit/test_service_workspace.py @@ -17,6 +17,7 @@ from fastraml.errors import ErrorKind, RamlError from fastraml.gctuning import tuned_gc from fastraml.registry import Raml +from fastraml.service.queries import folding_ranges_of from fastraml.service.workspace import _HEAD_BYTES, BOM, Workspace, _head, canonical from fastraml.uris import path_to_file_uri from fastraml.yamlnode import decode_source, read_head @@ -176,6 +177,38 @@ def load(self, uri, *, max_bytes=None): assert workspace._is_root(canonical(uri)) +class TestSourceTrees: + """The composed tree of a file's current text, once per text (docs/21 § 4).""" + + STRUCTURE = API + 'types:\n Thing:\n properties:\n name: string\n' + + def test_a_unchanged_text_composes_once(self, tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': self.STRUCTURE}) + uri = f'{folder}/api.raml' + workspace.open(uri, self.STRUCTURE, 1) + first = workspace.source(uri) + assert first is not None + assert workspace.source(uri) is first + + def test_a_changed_text_composes_again(self, tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': self.STRUCTURE}) + uri = f'{folder}/api.raml' + workspace.open(uri, self.STRUCTURE, 1) + first = workspace.source(uri) + workspace.change(uri, self.STRUCTURE + ' extra:\n properties:\n age: integer\n', 2) + second = workspace.source(uri) + assert second is not first + assert len(folding_ranges_of(second)) > len(folding_ranges_of(first)) + + def test_a_closed_buffer_composes_again(self, tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': self.STRUCTURE}) + uri = f'{folder}/api.raml' + workspace.open(uri, self.STRUCTURE, 1) + first = workspace.source(uri) + workspace.close(uri) + assert workspace.source(uri) is not first + + class TestSnapshots: FILES = { # noqa: RUF012 - read once per test 'api.raml': API + 'uses:\n lib: lib.raml\ntypes:\n Admin: lib.User\n', From 470030e5e34071edc29fc0b733fb77676a4f2343 Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 14:45:02 +0300 Subject: [PATCH 6/8] docs: record the simple source-rework with its first-use numbers --- docs/12-performance.md | 12 +- docs/15-implementation-plan.md | 19 ++- docs/21-language-service.md | 4 + .../service-source-simple-rework.md | 135 ++++++++++++++++++ 4 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 docs/reports/2026-10-10/service-source-simple-rework.md diff --git a/docs/12-performance.md b/docs/12-performance.md index 1a55f55..b78da8f 100644 --- a/docs/12-performance.md +++ b/docs/12-performance.md @@ -215,11 +215,13 @@ A service workload's measured region is a cold snapshot: the parse, plus the first query of each kind, then reuse of the built indices. Read its delta in two parts. The parse side is where the base revision built and retained every file's tree and the comparison does not. The first-use side is where a query -composes on demand the tree the base kept for free, one composition per file -per snapshot, about a third of a parse of the file; repeated queries of the -same kind are cached and show no delta. When a change moves composition from -parse time to first use, name which side the delta is on before attributing -it to the change's hot path (docs/21 § 2, docs/21 § 4). +composes on demand the tree the base retained at parse time, one composition +per file per snapshot, about a third of a parse of the file; repeated +queries of the same kind are cached and show no delta. The first-use cost is +avoidable only by retaining the trees, the per-snapshot memory cost +(docs/15 § 2). When a change moves composition from parse time to first use, +name which side the delta is on before attributing it to the change's hot +path (docs/21 § 2, docs/21 § 4). `bench compare` against the committed baseline is only a coarse check for large regressions: the baseline and the comparison run at different times, and diff --git a/docs/15-implementation-plan.md b/docs/15-implementation-plan.md index f6d38e6..526f62b 100644 --- a/docs/15-implementation-plan.md +++ b/docs/15-implementation-plan.md @@ -62,12 +62,15 @@ the media-type fixes and the type-walk work landed together: - `hover`/`unwrap` and `inlays`/`unwrap` (measured against `2b6502e` when the service's snapshots stopped retaining parse trees): time +27.1% (noise 3.3%) and +21.9% (noise 4.8%), peak +16.6% and +12.9%, kept +4.0% and - +3.3%. The cost is one composition of the queried file per snapshot, paid - by the first hover or inlay that needs its tree (docs/21 § 2); repeated - queries are cached, and the edit loop is within noise on time, peak - -20.5%, kept -24.4% (docs/21 § 4). Revisit: a clean file is composed - twice per snapshot, once for the hover index and once for the workspace's - source cache; sharing one tree between them removes the second. + +3.3%. Accepted for the memory the retention costs: the edit loop + (`large/service`) is within noise on time, peak -20.5%, kept -24.4%, and + `effective-types` keeps -27.5%. The cost is one composition of the queried + file per snapshot, about a third of a parse of it, paid by the first hover + or inlay that needs the tree (docs/21 § 2); repeated queries are cached. + The alternative is retaining the trees at parse time, the per-snapshot + memory cost. Revisit: a clean file is composed twice per snapshot, once + for the hover index and once for the workspace's source cache (docs/21 § 4); + sharing one tree between them removes the second. ## 3. Potential future work @@ -78,7 +81,9 @@ parser rules without an owning design document and tests. The language service and its LSP are built ([21](21-language-service.md)). The ordered plan for the rest, completion, editing features and MCP, is [research/language-service-plan.md](research/language-service-plan.md): M1 to -M4 are done, and completion (M5) comes next. Its design records are +M4 are done, and completion (M5) comes next, with its prerequisites in: the +resolver's visible-names enumeration (docs/04 § 2) and the workspace's +composed source cache (docs/21 § 4). Its design records are [archive/language-server.md](archive/language-server.md) and [archive/language-service-architecture.md](archive/language-service-architecture.md); its parser prerequisite, a trustworthy `parse_lenient` model, is analysed in diff --git a/docs/21-language-service.md b/docs/21-language-service.md index cd45cb6..dfdfe5f 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -55,6 +55,10 @@ the TCK's 1011 files. `parser:` limits. It keeps the YAML trees too, `retain_source=True`, only when an enabled lint rule reads them; no rule in the default set does. Decoders record the small syntax facts that `deprecated-schemas` reads (docs/18 § 6). +The first query that needs a file's tree composes it from the retained text, +once per snapshot for hover and inlays (§ 4.2, § 4.4) and once per current +text for folding and selection (§ 4); a composition is about a third of a +parse of the file, and it is the price of not retaining the trees (docs/15 § 2). It records the set of files it read: every retained text, every fragment, and every include it tried, found or not. It is built when a query first asks for it and kept until one of those files changes. A file that diff --git a/docs/reports/2026-10-10/service-source-simple-rework.md b/docs/reports/2026-10-10/service-source-simple-rework.md new file mode 100644 index 0000000..1b1d253 --- /dev/null +++ b/docs/reports/2026-10-10/service-source-simple-rework.md @@ -0,0 +1,135 @@ +# Simple rework of the source-retention replacement + +Date: 2026-10-10. Working branch: `refactor/service-source-simple`, from +`2b6502e`. The original replacement was parked on +`refactor/service-authoring` at `bc4b49d` after the 2026-10-09 review +concluded it overreached: its acceptance benchmarks failed (large/service +time +39.0% against noise 3.2%, hover +44.4%), and the language-service +design it was meant to serve (docs/archive/language-server.md § 5) needs no +canonical-record machinery. + +## 1. What landed + +Five commits, each independently useful, none building the record +architecture: + +| Commit | Change | +|---|---| +| `3a3553c` | Shared value descent extracted to `fastraml/types/navigation.py` (parked-branch pick; representation-agnostic). | +| `2873223` | Post-parse include reads no longer enter the parse's text store (`fastraml/parser/includes.py`); restores the docs/20 § 2 contract that a file is decoded at most once per parse. | +| `2e724b5` | The deprecated `schemas:`/`schema:` spellings are recorded in `Raml.syntax_aliases` as they are decoded; `deprecated-schemas` reads the records, so no rule in the default set requires retained trees and service snapshots parse text-only by default (docs/21 § 2). | +| `885a16f` | `visible_names`: a fragment resolver's local and directly imported declaration candidates, for completion (docs/04 § 2). | +| `efd9630` | `Workspace.source`: a file's current text composed once per text and held by the workspace, invalidated on change, close or disk change; folding and selection serve from it (docs/21 § 4). New `source-structure` feature workload with its reach test. | + +## 2. First-use mechanics + +The service composes a file's YAML tree in two independent caches, each +built lazily on first interaction: + +- The hover index (`fastraml/service/hover.py`, `_node`) composes the + file's snapshot text into a per-snapshot cache. The first hover or inlay + that needs the tree pays it: any field hover reaches it through the + field documentation, and a hover in a literal include additionally + triggers the one-time context discovery that composes every fragment + file. +- The workspace source cache (`fastraml/service/workspace.py`, + `Workspace.source`) composes the file's current text into a per-buffer + cache. The first folding or selection request for that text pays it. + +Cost, measured on the hover corpus at 400 families (entry 304 KiB, +13,204 lines): one `compose` is 60.1 ms best of 5 (61.1, 60.7, 62.9, +62.3, 60.1), against a full unwrap+validate parse of the same file at +158.8 ms best of 5. One composition is about 38 percent of one parse, or +0.2 ms per KiB. Nothing is built eagerly; the cache is built by whichever +interaction first needs the tree. + +## 3. Numbers + +`bench ab 2b6502e`, 3 rounds, best of 3 per round, at `efd9630`: + +| Workload | A ms | B ms | time | noise | peak | kept | +|---|---:|---:|---:|---:|---:|---:| +| `hover/unwrap` | 274.8 | 349.4 | +27.1% | 3.3% | +16.6% | +4.0% | +| `inlays/unwrap` | 291.1 | 354.9 | +21.9% | 4.8% | +12.9% | +3.3% | +| `effective-types/unwrap` | 209.9 | 212.7 | noise | 2.6% | noise | -27.5% | +| `large/service` | 474.7 | 481.0 | noise | 8.9% | -20.5% | -24.4% | + +New workload, no base number, so linearity (`--repeat 3`): +`source-structure/unwrap` time 1.004 (113.4 / 56.5 ms), peak 0.998, +retained 1.000. + +General parse workloads are untouched: `large` across all seven non-service +configurations, and `endpoints` and `datatype-fragments` on parse, unwrap +and unwrap+lint, are all within noise, with memory identical to the byte in +most rows (same A/B conditions). The code agrees with the measurement: the +include text-store fix touches only `load_include`, which join calls and no +bench workload does; the deprecated-spelling recording fires only on +`schemas:`/`schema:` spellings that no bench corpus uses; and the lint +rule's record read, against the base's retained-tree walk, is within noise on +the lint configurations. + +Attribution: at `2e724b5` (before the last two commits) the same A/B +already shows hover +25.6% and inlays +23.5%, and comparing `2e724b5` to +`efd9630` moves neither. The entire first-use delta is `2e724b5`'s: +snapshots stopped retaining parse trees, so the first hover or inlay per +snapshot composes the file on demand. The deltas match one composition +each (hover +74.6 ms, inlays +63.8 ms, against 60 ms of compose). + +## 4. Interpreting service-workload deltas + +The parked branch's number misled because the workload lumps three costs +into one number. A service request's cost decomposes as: + +1. The parse: the snapshot rebuild, paid per edit. +2. First use: one composition per file per snapshot, paid by whichever + interaction needs the tree first. +3. Repeated use: cached, about zero. + +Against that, the three designs differ on two axes: + +| | parse side | first use | repeated use | +|---|---|---|---| +| `2b6502e` (retained trees) | composes every file's tree and keeps them | free | hover/inlay cached; folding and selection re-compose per request | +| parked `bc4b49d` (records) | builds a canonical record for every file, plus the model | projects a tree from the record, more than a compose | cached from the record | +| `efd9630` (this branch) | lighter than the base | one text composition, about 38% of a parse | everything cached | + +The parked branch regressed the parse side (paid on every edit, which is +why large/service moved +39.0%) and the first-use side (projection is +dearer than compose). This branch regresses only the first-use side, by +exactly one composition, and improves the parse side's memory. The hover +workload's cold-snapshot convention (docs/12 § 4) is the right place for +that first-use number, because an editor session rebuilds the snapshot on +every edit, so first use recurs once per file per edit. The repeated-use +side is not A/B-able — no existing workload drives folding or selection +repeatedly, and a new workload has no base number by docs/12 § 5 — so the +branch pins it instead: the `source-structure` reach test counts `compose` +calls and requires each measured pass to compose the text once, shared by +fifteen requests. + +Residual inefficiency: a clean file is composed twice per snapshot, once +from the snapshot text for the hover index and once from the buffer text +for the workspace cache. Sharing one tree between them when the texts are +identical is the first revisit candidate (docs/15 § 2). + +## 5. Decision + +Keep `2e724b5`. The trade is one first-use composition per file per +snapshot (about 60 ms on the 304 KiB corpus entry, one time per edit) for +a persistent memory reduction in the long-running server: large/service +peak -20.5%, kept -24.4%, and effective-types kept -27.5%, with the edit +loop's time within noise. Accepted with the revisit note in docs/15 § 2; +the interpretation rule above is recorded in docs/12 § 5. + +## 6. Verification + +Gate (`uv run ruff check . && uv run ruff format --check . && uv run mypy +fastraml/ && uv run pytest -q`) green at `efd9630`. Measurement commands: + +```bash +uv run python -m bench ab 2b6502e --bench hover --bench inlays --bench effective-types --config unwrap --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench large --config service --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench large --config parse --config unwrap --config validate --config unwrap+validate --config unwrap+graph --config unwrap+lint --config unwrap+occurrences --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench endpoints --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench datatype-fragments --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 +uv run python -m bench linearity --bench source-structure --repeat 3 +``` From fb0fb731a58feac5000aceac024ad74767423130 Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 16:01:24 +0300 Subject: [PATCH 7/8] test: carry the deprecated spellings in the general corpora MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parse-side compatibility recording (docs/18 § 6) only runs on `schemas:` and `schema:`, which no bench corpus used, so the parse workloads never measured it. Every other `large` library now declares its whole table under the `schemas:` alias (a fragment takes one table or the other), every other inherited type uses the `schema:` facet, the `large` root carries a `schemas:` table of its own, and `endpoints` declares its payload type in a `schemas:` table with every fourth body typed by `schema:`. A reach test pins that both corpora record both spellings. Bench (ab 2b6502e, 3 rounds, on the spelling-carrying corpora): time within noise on large (all eight configurations), endpoints and small (parse, unwrap, unwrap+lint). Allocation peak and kept move +0.4 to +0.8 percent on the configurations that hold the model — the syntax_aliases records themselves, about 1.3 thousand of them, roughly 120 KB on large. large/service: time noise, peak -20.1%, kept -24.0%. --- bench/corpus.py | 40 ++++++++++++++++--- docs/15-implementation-plan.md | 5 ++- .../service-source-simple-rework.md | 40 ++++++++++++------- tests/bench/test_corpus.py | 17 ++++++++ 4 files changed, 80 insertions(+), 22 deletions(-) diff --git a/bench/corpus.py b/bench/corpus.py index 7637f66..fa5f526 100644 --- a/bench/corpus.py +++ b/bench/corpus.py @@ -215,6 +215,19 @@ def _derived(name: str, _ordinal: int, previous: str) -> str: ) +def _derived_schema(name: str, _ordinal: int, previous: str) -> str: + """`_derived` under the deprecated `schema:` spelling (docs/18 § 6).""" + return ( + f' {name}:\n' + f' schema: {previous}\n' + f' properties:\n' + f' label?: string\n' + f' weight:\n' + f' type: number\n' + f' minimum: 0\n' + ) + + def _array(name: str, _ordinal: int, previous: str) -> str: return f' {name}: {previous}[]\n' @@ -262,7 +275,12 @@ def _type_block(ordinal: int, previous: str | None) -> str: name = f'T{ordinal}' if previous is None: return _plain(name, ordinal, '') - return _FORMS[ordinal % len(_FORMS)](name, ordinal, previous) + form = _FORMS[ordinal % len(_FORMS)] + # Every other derived type takes the deprecated `schema:` spelling, so the + # parse exercises the compatibility recording (docs/18 § 6). + if form is _derived and ordinal % 12 == 1: + form = _derived_schema + return form(name, ordinal, previous) _COMMON = """#%RAML 1.0 Library @@ -287,10 +305,13 @@ def _library(index: int, type_count: int, *, depth: int) -> str: *same* file through a different spelling of the path. That is the diamond the compose cache exists for (docs/12 § 1); if canonicalisation ever regresses, this corpus goes quadratic and the linearity check catches it. + Every other library declares its whole table under the deprecated + `schemas:` alias, as a pre-1.0 document would: a fragment takes + `schemas:` or `types:`, never both (docs/18 § 6). """ lines = ['#%RAML 1.0 Library', f'usage: generated library {index}', 'uses:'] lines.append(f' common: {"../" * depth}common.raml') - lines.append('types:') + lines.append('schemas:' if index % 2 else 'types:') previous: str | None = None for ordinal in range(type_count): lines.append(_type_block(ordinal, previous).rstrip('\n')) @@ -307,6 +328,8 @@ def _api_using(libraries: list[str]) -> str: 'uses:', ] lines.extend(f' lib{index}: {path}' for index, path in enumerate(libraries)) + lines.append('schemas:') + lines.extend(_plain(f'Root{ordinal}', ordinal, '').rstrip('\n') for ordinal in range(3)) return '\n'.join(lines) + '\n' @@ -327,7 +350,11 @@ def write_large(root: Path, *, type_count: int = 7000, library_count: int = 150) """`type_count` types spread over `library_count` libraries. Sized after go-raml's published corpus (7124 types, 148 libraries). Halve - `type_count` and `library_count` together for the linearity check. + `type_count` and `library_count` together for the linearity check. Every + other library declares its types under the deprecated `schemas:` alias, + every other inherited type uses the deprecated `schema:` facet, and the + root carries a `schemas:` table of its own, so the parse exercises the + compatibility recording (docs/18 § 6). """ per_library, remainder = divmod(type_count, library_count) files = {'common.raml': _COMMON} @@ -375,12 +402,15 @@ def write_endpoints(root: Path, *, resource_count: int = 500) -> Path: Measures the two-stage build (docs/12 § 1): 6000 method-level trait applications, every one of them a tree merge rather than a model merge. + `Item` is declared in a deprecated `schemas:` table, and every fourth + resource types its body with the deprecated `schema:` facet, so the parse + exercises the compatibility recording (docs/18 § 6). """ lines = [ '#%RAML 1.0', 'title: Generated endpoint benchmark', 'baseUri: https://example.test', - 'types:', + 'schemas:', ' Item:', ' type: object', ' properties:', @@ -398,7 +428,7 @@ def write_endpoints(root: Path, *, resource_count: int = 500) -> Path: lines.append(' 200:') lines.append(' body:') lines.append(' application/json:') - lines.append(' type: Item') + lines.append(' schema: Item' if index % 4 == 0 else ' type: Item') _write(root, {'api.raml': '\n'.join(lines) + '\n'}) return root / 'api.raml' diff --git a/docs/15-implementation-plan.md b/docs/15-implementation-plan.md index 526f62b..9ee48bb 100644 --- a/docs/15-implementation-plan.md +++ b/docs/15-implementation-plan.md @@ -63,8 +63,9 @@ the media-type fixes and the type-walk work landed together: service's snapshots stopped retaining parse trees): time +27.1% (noise 3.3%) and +21.9% (noise 4.8%), peak +16.6% and +12.9%, kept +4.0% and +3.3%. Accepted for the memory the retention costs: the edit loop - (`large/service`) is within noise on time, peak -20.5%, kept -24.4%, and - `effective-types` keeps -27.5%. The cost is one composition of the queried + (`large/service`) is within noise on time, peak -20.1%, kept -24.0% + (measured on the spelling-carrying corpora), and `effective-types` keeps + -27.5%. The cost is one composition of the queried file per snapshot, about a third of a parse of it, paid by the first hover or inlay that needs the tree (docs/21 § 2); repeated queries are cached. The alternative is retaining the trees at parse time, the per-snapshot diff --git a/docs/reports/2026-10-10/service-source-simple-rework.md b/docs/reports/2026-10-10/service-source-simple-rework.md index 1b1d253..a55809a 100644 --- a/docs/reports/2026-10-10/service-source-simple-rework.md +++ b/docs/reports/2026-10-10/service-source-simple-rework.md @@ -10,7 +10,7 @@ canonical-record machinery. ## 1. What landed -Five commits, each independently useful, none building the record +Six commits, each independently useful, none building the record architecture: | Commit | Change | @@ -20,6 +20,7 @@ architecture: | `2e724b5` | The deprecated `schemas:`/`schema:` spellings are recorded in `Raml.syntax_aliases` as they are decoded; `deprecated-schemas` reads the records, so no rule in the default set requires retained trees and service snapshots parse text-only by default (docs/21 § 2). | | `885a16f` | `visible_names`: a fragment resolver's local and directly imported declaration candidates, for completion (docs/04 § 2). | | `efd9630` | `Workspace.source`: a file's current text composed once per text and held by the workspace, invalidated on change, close or disk change; folding and selection serve from it (docs/21 § 4). New `source-structure` feature workload with its reach test. | +| `661e063` | The general corpora carry the deprecated spellings (docs/18 § 6), so the parse workloads exercise the compatibility recording, pinned by a reach test. | ## 2. First-use mechanics @@ -45,28 +46,35 @@ interaction first needs the tree. ## 3. Numbers -`bench ab 2b6502e`, 3 rounds, best of 3 per round, at `efd9630`: +`bench ab 2b6502e`, 3 rounds, best of 3 per round, at `efd9630`; the +`large/service` row is re-measured on the spelling-carrying corpora of the +last commit: | Workload | A ms | B ms | time | noise | peak | kept | |---|---:|---:|---:|---:|---:|---:| | `hover/unwrap` | 274.8 | 349.4 | +27.1% | 3.3% | +16.6% | +4.0% | | `inlays/unwrap` | 291.1 | 354.9 | +21.9% | 4.8% | +12.9% | +3.3% | | `effective-types/unwrap` | 209.9 | 212.7 | noise | 2.6% | noise | -27.5% | -| `large/service` | 474.7 | 481.0 | noise | 8.9% | -20.5% | -24.4% | +| `large/service` | 476.0 | 474.0 | noise | 6.1% | -20.1% | -24.0% | New workload, no base number, so linearity (`--repeat 3`): `source-structure/unwrap` time 1.004 (113.4 / 56.5 ms), peak 0.998, retained 1.000. -General parse workloads are untouched: `large` across all seven non-service -configurations, and `endpoints` and `datatype-fragments` on parse, unwrap -and unwrap+lint, are all within noise, with memory identical to the byte in -most rows (same A/B conditions). The code agrees with the measurement: the -include text-store fix touches only `load_include`, which join calls and no -bench workload does; the deprecated-spelling recording fires only on -`schemas:`/`schema:` spellings that no bench corpus uses; and the lint -rule's record read, against the base's retained-tree walk, is within noise on -the lint configurations. +General parse workloads, on corpora that carry the deprecated spellings +(pinned by `test_general_corpora_exercise_the_deprecated_spellings`): every +other `large` library declares its types under the `schemas:` alias, every +other inherited type uses the `schema:` facet, and `endpoints` declares its +payload type in a `schemas:` table with every fourth body typed by +`schema:`. With the spellings in the input, `large` across all eight +configurations, and `endpoints` and `small` on parse, unwrap and +unwrap+lint, are within noise on time. Allocation peak and kept move +0.4 to ++0.8 percent on the configurations that hold the model — the +`syntax_aliases` records themselves, about 1.3 thousand of them, roughly +120 KB on `large` — and are noise where the model is dropped. The include +text-store fix still touches only `load_include`, which join calls and no +bench workload does, and the lint rule's record read, against the base's +retained-tree walk, is within noise on the lint configurations. Attribution: at `2e724b5` (before the last two commits) the same A/B already shows hover +25.6% and inlays +23.5%, and comparing `2e724b5` to @@ -116,9 +124,10 @@ identical is the first revisit candidate (docs/15 § 2). Keep `2e724b5`. The trade is one first-use composition per file per snapshot (about 60 ms on the 304 KiB corpus entry, one time per edit) for a persistent memory reduction in the long-running server: large/service -peak -20.5%, kept -24.4%, and effective-types kept -27.5%, with the edit -loop's time within noise. Accepted with the revisit note in docs/15 § 2; -the interpretation rule above is recorded in docs/12 § 5. +peak -20.1%, kept -24.0% (measured on the spelling-carrying corpus), and +effective-types kept -27.5%, with the edit loop's time within noise. +Accepted with the revisit note in docs/15 § 2; the interpretation rule +above is recorded in docs/12 § 5. ## 6. Verification @@ -130,6 +139,7 @@ uv run python -m bench ab 2b6502e --bench hover --bench inlays --bench effective uv run python -m bench ab 2b6502e --bench large --config service --rounds 3 --repeat 3 uv run python -m bench ab 2b6502e --bench large --config parse --config unwrap --config validate --config unwrap+validate --config unwrap+graph --config unwrap+lint --config unwrap+occurrences --rounds 3 --repeat 3 uv run python -m bench ab 2b6502e --bench endpoints --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 +uv run python -m bench ab 2b6502e --bench small --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 uv run python -m bench ab 2b6502e --bench datatype-fragments --config parse --config unwrap --config unwrap+lint --rounds 3 --repeat 3 uv run python -m bench linearity --bench source-structure --repeat 3 ``` diff --git a/tests/bench/test_corpus.py b/tests/bench/test_corpus.py index 2dc6bec..e65ca63 100644 --- a/tests/bench/test_corpus.py +++ b/tests/bench/test_corpus.py @@ -903,6 +903,23 @@ def test_large_reaches_common_by_two_spellings(tmp_path): assert len(common) == 1 +def test_general_corpora_exercise_the_deprecated_spellings(tmp_path): + """The parse-side compatibility recording only runs on `schemas:` and `schema:`. + + The general corpora carry both spellings (docs/18 § 6), so the parse + workloads measure a document that records them. If a writer ever drops a + spelling, the recording goes unmeasured and this test says so. + """ + corpora = ( + ('large', corpus.write_large(tmp_path / 'large', type_count=24, library_count=4)), + ('endpoints', corpus.write_endpoints(tmp_path / 'endpoints', resource_count=3)), + ) + for name, entry in corpora: + raml = parse_from_path(entry) + spelled = {use.name for uses in raml.syntax_aliases.values() for use in uses} + assert spelled == {'schemas', 'schema'}, f'{name} corpus records both deprecated spellings' + + class TestBaselinesMerge: """`baseline --bench small` must not delete the other four benches' rows. From 42c434e047e7f2983303a914d9644b47700a3306 Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 10 Oct 2026 21:37:46 +0300 Subject: [PATCH 8/8] perf: share queried source trees across service consumers Use one workspace-owned plain tree for compatible hover/inlay and folding/selection inputs. Cache failures and include depth/backend policy in compatibility. Snapshots borrow the owner weakly; stale input misses cannot replace current entries, while unrelated edits and unchanged dependencies can reuse source. Preserve full-source originals and separate JSON normalization. Against 94ebc60, service-session/unwrap time -12.5% beyond 2.8% noise, peak -26.5%, kept -16.9%; source-first time within 22.0% noise, peak -22.7%, kept -16.9%. All three relevant linearity checks pass. The Windows gate passes: 5838 passed, 68 skipped, 1 xfailed. The user accepts the remaining cold-query and post-query allocation trade-off for merge-then-recover. --- docs/15-implementation-plan.md | 24 +-- docs/21-language-service.md | 37 +++- .../service-baseline-integration.md | 66 +++++++ .../2026-10-10/service-recovery-plan.md | 7 +- fastraml/service/hover.py | 43 ++++- fastraml/service/source.py | 107 +++++++++++ fastraml/service/workspace.py | 64 +++++-- tests/bench/test_corpus.py | 6 +- tests/bench/test_service_session.py | 3 +- tests/unit/test_service_source_cache.py | 179 ++++++++++++++++++ 10 files changed, 487 insertions(+), 49 deletions(-) create mode 100644 fastraml/service/source.py create mode 100644 tests/unit/test_service_source_cache.py diff --git a/docs/15-implementation-plan.md b/docs/15-implementation-plan.md index 0c8490d..840c0c9 100644 --- a/docs/15-implementation-plan.md +++ b/docs/15-implementation-plan.md @@ -20,7 +20,9 @@ that each extend the same master remains deferred (docs/19 § 7). XML Schema external types are unsupported ([01](01-scope-and-coverage.md) § 3). **Service baseline acceptance and recovery.** The simpler source-cache branch -is undergoing comparison with master using the portable mixed-request workloads. +passes the master-controlled rebuild and mixed-request checks after sharing its +query source owner. The user accepted its documented first-use trade-off; +CI verification and integration are pending. The record-backed representation replacement remains parked; its recurring rebuild and first-use costs are recorded in the [service cost review](reports/2026-10-10/service-authoring-cost-review.md). @@ -68,19 +70,13 @@ the media-type fixes and the type-walk work landed together: endpoint build still holds; skip the swap where the node is retained anyway. - `validate`: time up to +4.9% on some configurations in a 3-round run; a 5-round rerun of `unwrap+validate` was within noise. -- `hover`/`unwrap` and `inlays`/`unwrap` (measured against `2b6502e` when the - service's snapshots stopped retaining parse trees): time +27.1% (noise - 3.3%) and +21.9% (noise 4.8%), peak +16.6% and +12.9%, kept +4.0% and - +3.3%. Accepted for the memory the retention costs: the edit loop - (`large/service`) is within noise on time, peak -20.1%, kept -24.0% - (measured on the spelling-carrying corpora), and `effective-types` keeps - -27.5%. The cost is one composition of the queried - file per snapshot, about a third of a parse of it, paid by the first hover - or inlay that needs the tree (docs/21 § 2); repeated queries are cached. - The alternative is retaining the trees at parse time, the per-snapshot - memory cost. Revisit: a clean file is composed twice per snapshot, once - for the hover index and once for the workspace's source cache (docs/21 § 4); - sharing one tree between them removes the second. +- Text-only service snapshots pay an on-demand composition for a queried input + unless a compatible source entry is already ready (docs/21 § 4). The shared + owner removes the old duplicate hover/structural tree. The accepted baseline's + remaining first-use cost and post-query allocation trade-off are recorded in the + [integration report](reports/2026-10-10/service-baseline-integration.md). + Model-backed declaration inlays and cursor-local source lookup are the next + separately measured recovery candidates. ## 3. Potential future work diff --git a/docs/21-language-service.md b/docs/21-language-service.md index dfdfe5f..344871d 100644 --- a/docs/21-language-service.md +++ b/docs/21-language-service.md @@ -20,7 +20,8 @@ and several views, and only `cli/` imports it (`docs/02` § 2; | Module | Holds | |---|---| -| `service/workspace.py` | buffers, the overlay loader, roots, snapshots, staleness, composed source trees | +| `service/workspace.py` | buffers, the overlay loader, roots, snapshots and staleness | +| `service/source.py` | on-demand composed source shared by workspace and snapshot consumers | | `service/text.py` | converting a column between fastRAML and a protocol | | `service/queries.py` | the queries, in fastRAML positions | | `service/outline.py` | the outline, over the authorship view (`docs/16` § 10) | @@ -55,10 +56,12 @@ the TCK's 1011 files. `parser:` limits. It keeps the YAML trees too, `retain_source=True`, only when an enabled lint rule reads them; no rule in the default set does. Decoders record the small syntax facts that `deprecated-schemas` reads (docs/18 § 6). -The first query that needs a file's tree composes it from the retained text, -once per snapshot for hover and inlays (§ 4.2, § 4.4) and once per current -text for folding and selection (§ 4); a composition is about a third of a -parse of the file, and it is the price of not retaining the trees (docs/15 § 2). +The first query that needs a file's tree composes it on demand. Hover, inlays, +folding and selection share the workspace's source owner when their input and +composition policy agree (§ 4). Parsing does not populate that query cache or +retain all producer trees. Snapshot-local hover indices keep requested trees; +snapshots borrow the workspace's cache weakly rather than keep its other files +alive. Without a live owner, a held snapshot can compose its own retained text. It records the set of files it read: every retained text, every fragment, and every include it tried, found or not. It is built when a query first asks for it and kept until one of those files changes. A file that @@ -133,6 +136,26 @@ composed YAML and the parser's structural grammar. | `diagnostics` | `RamlError.chains()` and the lint findings | | `tree` | `build_tree` (`docs/16` § 6) as JSON text, only on an unwrapped model | +**Shared source.** `Workspace.source(uri)` composes once for its current input, +including a failed composition. Cache compatibility requires the canonical URI, +equal text after the parser's single leading-BOM removal, depth limit and YAML +backend. Equal text from another disk decode can reuse the same tree. Buffer +changes, close and watched disk changes discard that URI's entry; changing the +composition policy discards the cache. Unchanged dependency entries survive a +root edit. These trees carry no semantic binding or include receiving context; +those belong to the snapshot's hover index. + +A source-before-snapshot request and a later hover share their composed tree; +the parse itself still composes independently. An eligible original RAML/YAML +container already retained for a syntax-reading lint rule can replace an earlier +standalone owner on demand, without changing the older tree. JSON include +normalization and synthetic non-YAML/schema scalars are not that standalone +composition policy, so they are not substituted for generic source trees. +An old snapshot may borrow an identical ready entry, but a miss after that input +or its policy was invalidated remains private to that snapshot and cannot replace the +current workspace's entry. Query answers therefore keep their own input positions +without making an old view a source for current text. + **Stopped parses.** A query on a snapshot that stopped early answers from the stages it completed. A declaration is an occurrence after decoding; a type name used in an expression is one only once P7 bound it. No query raises on a @@ -456,6 +479,10 @@ could save. - `test_service_text.py`: conversion in each encoding, both ways. - `test_service_workspace.py`: roots, buffers over the disk, the sandbox, and when a snapshot is stale. +- `test_service_source_cache.py`: both source/semantic request orders, failed + composition, equal closed-file inputs, depth/backend compatibility, unchanged + dependencies and unrelated edits, private old-snapshot misses, weak owner + lifetime, retained original trees and distinct JSON-include normalization. - `test_service_queries.py`: each query on one document with a library, a DataType include, a trait and a resource type; every query on a parse stopped at each stage; an Extension's outline; and, over the TCK, that every diff --git a/docs/reports/2026-10-10/service-baseline-integration.md b/docs/reports/2026-10-10/service-baseline-integration.md index 90ff671..812809b 100644 --- a/docs/reports/2026-10-10/service-baseline-integration.md +++ b/docs/reports/2026-10-10/service-baseline-integration.md @@ -45,6 +45,21 @@ The production implementation is unchanged in this first comparison. Control: `d2b8a02`). Three alternating rounds, best of three, identical candidate-generated corpora: +The mixed corpus is one **311,308-byte (304.012 KiB), 13,204-line** RAML file: +400 families, 2,000 named types, 400 annotation types and 400 resources. Each +reported time covers **three complete edit/query cycles**, not one parse or one +editor request. Each cycle includes two outlines, two viewport inlay requests, +two folding requests, three hovers, eight selection positions, links, lens +enumeration, diagnostics and occurrences. The source-first case adds one folding +request before each snapshot. These request counts are a cache/reuse scenario, +not a measured client request trace. + +Original master recomposes for both folds and every selection: 11 compositions +per version, 33 over the snapshot-first sequence. The simpler candidate composes +three times per version (parse, hover/inlays and workspace source), nine overall. +Thus the aggregate is deliberately sensitive to repeated uncached source work; +it must not be quoted as individual interactive-request latency. + | Workload (`unwrap`) | Control ms | Candidate ms | Time/noise | Peak | Kept | |---|---:|---:|---|---:|---:| | `service-session` | 3,108.7 | 1,560.2 | -49.8%, noise 4.4% | +2.1% | +26.8% | @@ -77,6 +92,57 @@ compatibility, unchanged dependencies, old snapshots and source-cache release. Measure mixed requests and normal rebuilds separately after the fix. Remaining first-use trade-offs still require their own acceptance decision. +### 2.2 Shared-source results and acceptance + +The shared-source implementation supplies one on-demand plain tree to both +consumer groups. Invalidations track changes only for read/cached URIs, so an +unrelated watched file does not prevent a current snapshot from publishing a +cache miss. Policy changes invalidate publication separately. Fourteen focused +cases protect the named ownership/failure/compatibility decisions; the mixed +reach test requires six compositions over three versions in either order +(one semantic and one query composition per version). + +Isolated A/B against the measurement-integrated candidate `94ebc60`: + +| Workload | Time/noise | Peak | Kept | +|---|---|---:|---:| +| `service-session` | -12.5%, noise 2.8% | -26.5% | -16.9% | +| `service-source-first` | within 22.0% noise | -22.7% | -16.9% | + +Final master control is `d2b8a02`. Three rounds/best of three, identical inputs: + +| Workload/configuration | Master ms | Shared-source candidate ms | Time/noise | Peak | Kept | +|---|---:|---:|---|---:|---:| +| `large/service` | 451.0 | 454.9 | within 12.0% noise | -20.1% | -24.0% | +| `service-session/unwrap` | 3,078.7 | 1,307.7 | -57.5%, noise 8.3% | -25.0% | +5.4% | +| `service-source-first/unwrap` | 3,352.6 | 1,377.8 | -58.9%, noise 8.3% | -31.1% | +5.4% | +| `hover/unwrap` | 268.1 | 331.6 | +23.7%, noise 2.4% | +16.6% | +4.1% | +| `inlays/unwrap` | 279.3 | 343.7 | +23.0%, noise 12.3% | +13.0% | +3.4% | +| `effective-types/unwrap` | 209.2 | 208.9 | within 2.8% noise | +0.0% | -27.4% | + +The mixed candidate keeps 25.851 MB versus master's 24.521 MB after requests, +instead of the original candidate's 31.097 MB. There is no longer a second retained +full tree for these consumers. The remaining allocation difference is not claimed +to have been attributed exclusively to one model field or position store. + +`large` and `endpoints`, on `parse`, `unwrap+validate` and `unwrap+lint`, remain +within time noise (0.8–3.4%). Ordinary model-retaining rows add 0.5–0.8% peak and +0.7–0.8% kept allocation; full lint peaks add 0.1–0.2%, with findings-only retained +allocation within noise. These corpora include the candidate's deprecated +spellings on both sides. + +The Windows gate passes: **5,838 passed, 68 skipped, 1 xfailed**. Linearity passes +for `source-structure`, `service-session` and `service-source-first`: normalized +time 1.008 / 1.118 / 1.004, peak 0.998 / 0.993 / 0.984, kept 1.000 / 0.998 / 0.998. + +After being shown the remaining cold-query cost and +5.4% post-query retained +allocation, the user explicitly selected **“Merge, then recover”**. Acceptance is +therefore against the primary rebuild goal, improved complete mixed sequences, +lower diagnostic/peak memory and corrected ownership, with the lazy first-use +trade-off recorded rather than hidden. Linux/CI verification is still required +before the production merge. Model-backed inlays are the first recovery experiment +against that accepted baseline, without accepting record capture. + ## 3. Selective recovery Pending acceptance of the production baseline. The first planned experiment is diff --git a/docs/research/2026-10-10/service-recovery-plan.md b/docs/research/2026-10-10/service-recovery-plan.md index 7f34a3d..ca4fb2b 100644 --- a/docs/research/2026-10-10/service-recovery-plan.md +++ b/docs/research/2026-10-10/service-recovery-plan.md @@ -127,9 +127,10 @@ Current execution: (5,810 passed, 67 skipped, 1 xfailed) and both mixed-workload linearity checks. All GitHub checks passed, including Ubuntu, TCK and benchmarks; PR #1 merged into master as `d2b8a02`. -- Stage B: initial master-controlled mixed comparisons confirm faster queries but - expose duplicate retained hover/workspace trees. A bounded shared-source fix is - the next acceptance task; the production candidate is not yet accepted. +- Stage B: duplicate hover/workspace ownership is fixed and locally verified. + The user explicitly accepted the remaining first-use and +5.4% post-query + allocation trade-off with “Merge, then recover”; GitHub verification and the + production merge are next. Results and the decision are in the integration report. - Stage C: pending an accepted baseline. Update these entries as each stage completes. Keep measurements in the dated diff --git a/fastraml/service/hover.py b/fastraml/service/hover.py index e1e61ec..ea82472 100644 --- a/fastraml/service/hover.py +++ b/fastraml/service/hover.py @@ -32,6 +32,7 @@ type_label, underlying_type, ) +from fastraml.service.source import original_tree from fastraml.service.text import Lines from fastraml.types.base import BaseShape, Parameter, facets_of from fastraml.types.complex_ import ObjectShape, UnknownShape @@ -44,14 +45,16 @@ from fastraml.views import authored from fastraml.views.occurrences import DECLARATION_KINDS, Kind, Link, Occurrence, Occurrences, Role from fastraml.views.render import type_name -from fastraml.yamlnode import TAG_INCLUDE, TAG_STR, Node, NodeKind, compose, pairs +from fastraml.yamlnode import TAG_INCLUDE, TAG_STR, Node, NodeKind, backend_name, compose, pairs if TYPE_CHECKING: from collections.abc import Iterable, Iterator + from weakref import ReferenceType from fastraml.parser.endpoints import Body, EndPoint, Operation, Response from fastraml.positions import Position from fastraml.registry import Raml + from fastraml.service.source import Sources from fastraml.types.base import ScalarFacet from fastraml.types.expressions import ExprCache, RdtNode @@ -106,14 +109,29 @@ class Hover: '_responses', '_root', '_sites', + '_source_backend', '_source_builtins', + '_source_generation', + '_sources', '_subjects', ) - def __init__(self, raml: Raml, root: str, occurrences: Occurrences) -> None: + def __init__( # noqa: PLR0913 - the snapshot's borrowed source owner and composition policy + self, + raml: Raml, + root: str, + occurrences: Occurrences, + *, + sources: ReferenceType[Sources] | None = None, + source_generation: int = 0, + source_backend: str = '', + ) -> None: self._raml = raml self._root = root.rpartition('/')[0] + '/' self._occurrences = occurrences + self._sources = sources + self._source_generation = source_generation + self._source_backend = source_backend self._subjects: dict[int, _Subject] = {} self._sites: dict[tuple[str, int, int], _Subject] = {} self._mappings: dict[tuple[str, int, int], _Subject] = {} @@ -359,11 +377,22 @@ def _node(self, uri: str) -> Node | None: if uri in self._nodes: return self._nodes[uri] node = self._raml.source_nodes.get(uri) or self._raml.include_nodes.get(uri) - if node is None: - text = self._raml.source_texts.get(uri) - if text is not None: - with suppress(RamlError): - node = compose(text, uri=uri, max_depth=self._raml.max_depth) + text = self._raml.source_texts.get(uri) + shared = None if self._sources is None else self._sources() + if node is not None and original_tree(uri, node) is None: + self._nodes[uri] = node + return node + if text is not None and shared is not None: + node = shared.node( + uri, + text, + max_depth=self._raml.max_depth, + generation=self._source_generation, + original=node if self._source_backend == backend_name() else None, + ) + elif node is None and text is not None: + with suppress(RamlError): + node = compose(text, uri=uri, max_depth=self._raml.max_depth) self._nodes[uri] = node return node diff --git a/fastraml/service/source.py b/fastraml/service/source.py new file mode 100644 index 0000000..24d23e5 --- /dev/null +++ b/fastraml/service/source.py @@ -0,0 +1,107 @@ +"""On-demand composed trees shared by current-text and snapshot consumers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from fastraml.errors import RamlError +from fastraml.yamlnode import NodeKind, backend_name, compose + +if TYPE_CHECKING: + from fastraml.yamlnode import Node + +_BOM = chr(0xFEFF) + + +def original_tree(uri: str, root: Node | None) -> Node | None: + """Only unchanged YAML/RAML containers have the standalone composition policy. + + JSON includes can normalize whitespace; non-YAML includes and external + schemas can use synthetic scalars. Neither is a generic composed tree. + """ + path = uri.split('#', 1)[0].split('?', 1)[0] + if root is not None and root.kind is not NodeKind.SCALAR and path.lower().endswith(('.raml', '.yaml', '.yml')): + return root + return None + + +@dataclass(slots=True, eq=False) +class _Entry: + text: str + max_depth: int + backend: str + root: Node | None + + +class Sources: + """Workspace-owned trees; snapshots borrow this owner rather than retain it.""" + + __slots__ = ('__weakref__', '_changed', '_entries', '_policy_generation', 'backend', 'generation', 'max_depth') + + def __init__(self, max_depth: int) -> None: + self._entries: dict[str, _Entry] = {} + self._changed: dict[str, int] = {} + self._policy_generation = 0 + self.max_depth = max_depth + self.backend = backend_name() + self.generation = 0 + + def configure(self, max_depth: int) -> None: + """The current owner's composition policy invalidates older publications.""" + backend = backend_name() + if (max_depth, backend) != (self.max_depth, self.backend): + self._entries.clear() + self.generation += 1 + self._policy_generation = self.generation + self.max_depth, self.backend = max_depth, backend + + def discard(self, uri: str, *, read: bool) -> None: + cached = self._entries.pop(uri, None) + self.generation += 1 + # Watchers see unrelated files too. Remember changes only for source + # a snapshot/cache has used, including any already tracked old reader. + if read or cached is not None or uri in self._changed: + self._changed[uri] = self.generation + + def lookup(self, uri: str, text: str, max_depth: int) -> _Entry | None: + """An equal parsed input/policy can share a successful or failed result.""" + entry = self._entries.get(uri) + if ( + entry is not None + and entry.text == text.removeprefix(_BOM) + and entry.max_depth == max_depth + and entry.backend == backend_name() + ): + return entry + return None + + def node( + self, + uri: str, + text: str, + *, + max_depth: int, + generation: int | None = None, + original: Node | None = None, + ) -> Node | None: + """Compose on a miss; an old snapshot cannot replace the current owner.""" + entry = self.lookup(uri, text, max_depth) + text = text.removeprefix(_BOM) + backend = backend_name() + current = ( + (generation is None or generation >= max(self._changed.get(uri, 0), self._policy_generation)) + and max_depth == self.max_depth + and backend == self.backend + ) + if entry is not None and not (original is not None and current): + return entry.root + root = original + if root is None: + try: + root = compose(text, uri=uri, max_depth=max_depth) + except RamlError: + root = None + if current: + self._entries[uri] = _Entry(text, max_depth, backend, root) + return root diff --git a/fastraml/service/workspace.py b/fastraml/service/workspace.py index 3b3b9d2..7755947 100644 --- a/fastraml/service/workspace.py +++ b/fastraml/service/workspace.py @@ -24,6 +24,7 @@ from dataclasses import dataclass, field from fnmatch import fnmatch from typing import TYPE_CHECKING, Final +from weakref import ref from fastraml.errors import ErrorKind, RamlError from fastraml.loaders import SafeFileLoader @@ -31,14 +32,16 @@ from fastraml.parser.fragments import FragmentKind, identify_fragment from fastraml.service.hover import Hover from fastraml.service.lenses import EffectiveViews +from fastraml.service.source import Sources, original_tree from fastraml.service.text import Lines from fastraml.uris import file_uri_to_path, path_to_file_uri, relative_to from fastraml.views.lint import configured_linter from fastraml.views.occurrences import build_occurrences -from fastraml.yamlnode import compose, read_head +from fastraml.yamlnode import backend_name, read_head if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping, Sequence + from weakref import ReferenceType from fastraml.config import FastRamlConfig from fastraml.registry import Raml @@ -103,6 +106,9 @@ class Snapshot: _findings: list[Finding] | None = field(default=None, repr=False) _hover: Hover | None = field(default=None, repr=False) _effective_views: EffectiveViews | None = field(default=None, repr=False) + sources: ReferenceType[Sources] | None = field(default=None, repr=False) + source_generation: int = field(default=0, repr=False) + source_backend: str = field(default='', repr=False) @property def effective_views(self) -> EffectiveViews | None: @@ -115,7 +121,14 @@ def hover(self) -> Hover | None: """The source and model indices used by author-facing hover.""" occurrences = self.occurrences if self._hover is None and self.raml is not None and occurrences is not None: - self._hover = Hover(self.raml, self.root, occurrences) + self._hover = Hover( + self.raml, + self.root, + occurrences, + sources=self.sources, + source_generation=self.source_generation, + source_backend=self.source_backend, + ) return self._hover @property @@ -181,9 +194,7 @@ def __init__( self.buffers: dict[str, Buffer] = {} self._roots: list[str] | None = None self._snapshots: dict[str, Snapshot] = {} - #: `uri` -> (the text it was composed from, the composed tree), one per - #: current text (docs/21 § 4). - self._sources: dict[str, tuple[str, Node]] = {} + self._sources = Sources(self.config.parser.max_depth) #: What each dropped snapshot read: the order `serving` tries roots in. self._last_read: dict[str, frozenset[str]] = {} #: Whether a snapshot was dropped since the last collection. @@ -242,15 +253,23 @@ def source(self, uri: str) -> Node | None: text = self.text(uri) if text is None: return None - cached = self._sources.get(uri) - if cached is not None and cached[0] is text: - return cached[1] - try: - root = compose(text, uri=uri) - except RamlError: - return None - self._sources[uri] = (text, root) - return root + depth = self.config.parser.max_depth + self._sources.configure(depth) + original = None + normalized = text.removeprefix(BOM) + for snapshot in self._snapshots.values(): + raml = snapshot.raml + if ( + raml is not None + and raml.retain_source + and raml.max_depth == depth + and snapshot.source_backend == self._sources.backend + and raml.source_texts.get(uri) == normalized + ): + original = original_tree(uri, raml.source_nodes.get(uri)) + if original is not None: + break + return self._sources.node(uri, text, max_depth=depth, original=original) def _put(self, uri: str, buffer: Buffer) -> None: """Hold `buffer`, and drop what read other text for `uri`. @@ -284,12 +303,14 @@ def _touched(self, uri: str, *, appeared: bool) -> None: A file that appeared may be one a failed include was looking for, so a snapshot that ended in an error is dropped then too. """ - self._sources.pop(uri, None) + read = False for root, snapshot in list(self._snapshots.items()): + read |= uri in snapshot.read if uri in snapshot.read or (appeared and snapshot.error is not None): del self._snapshots[root] self._last_read[root] = snapshot.read self._garbage = True + self._sources.discard(uri, read=read) # -- roots and snapshots -------------------------------------------------- @@ -368,6 +389,8 @@ def readers(self) -> dict[str, list[str]]: return found def _parse(self, root: str) -> Snapshot: + self._sources.configure(self.config.parser.max_depth) + source_backend = backend_name() disk = self._disk(root) if disk is None: refused = RamlError.new('path is outside the workspace root', root, info={'path': root}) @@ -395,7 +418,16 @@ def _parse(self, root: str) -> Snapshot: else RamlError.wrap('load resource', err, root, kind=ErrorKind.READING) ) return Snapshot(root, None, failure, frozenset({root})) - return Snapshot(root, raml, error, _read(raml, root), linter=self.linter) + return Snapshot( + root, + raml, + error, + _read(raml, root), + linter=self.linter, + sources=ref(self._sources), + source_generation=self._sources.generation, + source_backend=source_backend, + ) @property def linter(self) -> Linter: diff --git a/tests/bench/test_corpus.py b/tests/bench/test_corpus.py index 6188a79..912843c 100644 --- a/tests/bench/test_corpus.py +++ b/tests/bench/test_corpus.py @@ -133,14 +133,14 @@ def parse(entry, options): def test_source_structure_reaches_folding_and_selection_on_the_cached_tree(self, tmp_path, monkeypatch, count): from bench.__main__ import run_one from fastraml.service import queries - from fastraml.service import workspace as workspace_module + from fastraml.service import source as source_module folded = [] selected = [] compositions = 0 original_folding = queries.folding_ranges_of original_selection = queries.selection_ranges_of - original_compose = workspace_module.compose + original_compose = source_module.compose def folding(root): result = original_folding(root) @@ -161,7 +161,7 @@ def compose(text, **kwargs): monkeypatch.setattr(queries, 'folding_ranges_of', folding) monkeypatch.setattr(queries, 'selection_ranges_of', selection) - monkeypatch.setattr(workspace_module, 'compose', compose) + monkeypatch.setattr(source_module, 'compose', compose) entry = corpus.write_hover(tmp_path, family_count=count) run_one('source-structure', 'unwrap', entry, repeat=1) assert len(folded) == 2 * 3, 'timing and allocation both reach folding' diff --git a/tests/bench/test_service_session.py b/tests/bench/test_service_session.py index 3b3fb67..fb79827 100644 --- a/tests/bench/test_service_session.py +++ b/tests/bench/test_service_session.py @@ -58,6 +58,8 @@ def source(workspace, uri): snapshot = workspace.snapshot(prepared.root) if getattr(snapshot.raml, 'projection', None) is not None: assert len(compositions) == (6 if source_first else 3) + elif getattr(snapshot, 'sources', None) is not None: + assert len(compositions) == 6, 'each version shares one query composition in both request orders' else: assert len(compositions) >= 3, 'historical source readers may compose for each request' if hasattr(snapshot, 'outlines'): @@ -71,7 +73,6 @@ def source(workspace, uri): assert all(hints[index] == hints[index + 1] for index in (0, 2, 4)) assert counts['hovers'] == 9 assert counts['selections'] == 24 - assert len(workspace._snapshots) == 1 def test_session_can_query_an_unchanged_library_after_root_edits(tmp_path): diff --git a/tests/unit/test_service_source_cache.py b/tests/unit/test_service_source_cache.py new file mode 100644 index 0000000..4761da1 --- /dev/null +++ b/tests/unit/test_service_source_cache.py @@ -0,0 +1,179 @@ +"""One source input serves all consumers without retaining its workspace.""" + +import gc +from weakref import ref + +import pytest + +from fastraml.config import FastRamlConfig, ParserConfig +from fastraml.service import queries, source +from fastraml.service.workspace import BOM +from tests.unit.test_service_workspace import API, _workspace + +DOCUMENT = API + 'types:\n Item:\n type: string\n minLength: 2\n' + + +def _compositions(monkeypatch): + calls = [] + original = source.compose + + def compose(text, **kwargs): + calls.append((text, kwargs['max_depth'])) + return original(text, **kwargs) + + monkeypatch.setattr(source, 'compose', compose) + return calls + + +@pytest.mark.parametrize('source_first', [False, True]) +def test_hover_and_structure_share_one_tree_in_either_request_order(tmp_path, monkeypatch, source_first): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, BOM + DOCUMENT, 1) + calls = _compositions(monkeypatch) + first = workspace.source(uri) if source_first else None + snapshot = workspace.snapshot(uri) + assert snapshot.error is None + assert queries.hover(snapshot, uri, 6, 5) is not None + root = workspace.source(uri) + assert root is not None + assert snapshot.hover._node(uri) is root + assert not source_first or first is root + assert len(calls) == 1 + assert calls[0][0] == DOCUMENT, 'single-BOM normalization agrees with the parser input' + + +def test_failed_composition_is_cached_until_the_input_changes(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, API + 'broken: [\n', 1) + calls = _compositions(monkeypatch) + assert workspace.source(uri) is None + assert workspace.source(uri) is None + assert len(calls) == 1 + workspace.change(uri, DOCUMENT, 2) + assert workspace.source(uri) is not None + assert len(calls) == 2 + + +def test_closed_file_equal_text_reuses_the_tree_even_after_another_disk_decode(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + calls = _compositions(monkeypatch) + first = workspace.source(f'{folder}/api.raml') + assert workspace.source(f'{folder}/api.raml') is first + assert len(calls) == 1 + + +def test_source_composition_obeys_depth_and_does_not_reuse_a_different_limit(tmp_path, monkeypatch): + config = FastRamlConfig(parser=ParserConfig(max_depth=1)) + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}, config=config) + calls = _compositions(monkeypatch) + assert workspace.source(f'{folder}/api.raml') is None + assert workspace.source(f'{folder}/api.raml') is None + workspace.config = FastRamlConfig(parser=ParserConfig(max_depth=32)) + assert workspace.source(f'{folder}/api.raml') is not None + assert [depth for _, depth in calls] == [1, 32] + + +def test_source_cache_key_includes_the_yaml_backend(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + calls = _compositions(monkeypatch) + first = workspace.source(f'{folder}/api.raml') + current = source.backend_name() + monkeypatch.setattr(source, 'backend_name', lambda: 'python' if current == 'libyaml' else 'libyaml') + assert workspace.source(f'{folder}/api.raml') is not first + assert len(calls) == 2 + + +def test_old_snapshot_reads_do_not_evict_the_current_text_tree(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, DOCUMENT, 1) + old = workspace.snapshot(uri) + workspace.change(uri, DOCUMENT.replace('minLength: 2', 'minLength: 3'), 2) + current = workspace.source(uri) + calls = _compositions(monkeypatch) + assert old.hover._node(uri) is not current + assert workspace.source(uri) is current + assert len(calls) == 1, 'only the stale snapshot composes its old input' + + +def test_an_old_snapshot_cache_miss_cannot_publish_into_a_new_input_generation(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}) + uri = f'{folder}/api.raml' + workspace.open(uri, DOCUMENT, 1) + old = workspace.snapshot(uri) + workspace.change(uri, DOCUMENT + '# intermediate\n', 2) + workspace.change(uri, DOCUMENT, 3) + calls = _compositions(monkeypatch) + stale = old.hover._node(uri) + assert workspace.source(uri) is not stale + assert len(calls) == 2, 'the old miss is private even when current text returns to the same value' + + +def test_unchanged_dependency_structure_survives_a_root_snapshot_rebuild(tmp_path, monkeypatch): + library = '#%RAML 1.0 Library\ntypes:\n Item: string\n' + api = API + 'uses:\n lib: lib.raml\n' + workspace, folder = _workspace(tmp_path, {'api.raml': api, 'lib.raml': library}) + uri = f'{folder}/lib.raml' + workspace.open(uri, library, 1) + calls = _compositions(monkeypatch) + first = workspace.source(uri) + assert workspace.snapshot(f'{folder}/api.raml').hover._node(uri) is first + workspace.change(f'{folder}/api.raml', api + '# edit\n', 2) + assert workspace.snapshot(f'{folder}/api.raml').hover._node(uri) is first + assert len(calls) == 1 + + +def test_an_unrelated_change_does_not_prevent_a_current_snapshot_publishing_source(tmp_path, monkeypatch): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT, 'other.raml': API}) + uri = f'{folder}/api.raml' + snapshot = workspace.snapshot(uri) + workspace.change(f'{folder}/other.raml', API + '# changed\n', 1) + calls = _compositions(monkeypatch) + first = snapshot.hover._node(uri) + assert workspace.source(uri) is first + assert len(calls) == 1 + + +@pytest.mark.parametrize('source_first', [False, True]) +def test_full_source_retention_offers_its_original_tree_on_demand(tmp_path, monkeypatch, source_first): + config = FastRamlConfig(lint={'rules': [{'id': 'prefer-array-expression'}]}) + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT}, config=config) + uri = f'{folder}/api.raml' + calls = _compositions(monkeypatch) + standalone = workspace.source(uri) if source_first else None + snapshot = workspace.snapshot(uri) + original = snapshot.raml.source_nodes[uri] + assert workspace.source(uri) is original + assert snapshot.hover._node(uri) is original + assert len(calls) == int(source_first) + if source_first: + assert standalone is not original + assert queries.folding_ranges_of(standalone) == queries.folding_ranges_of(original) + + +def test_holding_a_snapshot_does_not_retain_the_workspace_source_cache(tmp_path): + workspace, folder = _workspace(tmp_path, {'api.raml': DOCUMENT, 'other.yaml': 'a:\n b: c\n'}) + snapshot = workspace.snapshot(f'{folder}/api.raml') + workspace.source(f'{folder}/other.yaml') + cache = ref(workspace._sources) + del workspace + gc.collect() + assert cache() is None + assert queries.hover(snapshot, f'{folder}/api.raml', 6, 5) is not None + + +def test_normalized_json_include_is_not_substituted_for_generic_current_source(tmp_path): + document = API + 'types:\n Item:\n properties: {a: integer}\n example: !include example.json\n' + config = FastRamlConfig(lint={'rules': [{'id': 'prefer-array-expression'}]}) + workspace, folder = _workspace( + tmp_path, {'api.raml': document, 'example.json': '\t{\n\t"a": 1\n}\n'}, config=config + ) + snapshot = workspace.snapshot(f'{folder}/api.raml') + assert snapshot.error is None + uri = f'{folder}/example.json' + original = snapshot.raml.include_nodes[uri] + assert workspace.source(uri) is None, 'generic YAML composition does not perform JSON whitespace normalization' + assert snapshot.hover._node(uri) is original + assert workspace.source(uri) is None