feat: end types at void, end values at a postfix !, and read delete as an operator

.changeset/delete-unary-keyword.md

@@ -0,0 +1,5 @@
+---
+"htmljs-parser": minor
+---
+
+Read `delete` as a prefix operator, like `typeof` and `void`, so `<div a=delete x.y/>` is one attribute value and a statement ending in `delete` continues onto the next line.

.changeset/empty-placeholder-error.md

@@ -0,0 +1,5 @@
+---
+"htmljs-parser": minor
+---
+
+An empty placeholder (`${}`) now stops at its error instead of also emitting an `onPlaceholder` whose end is past the input.

.changeset/trailing-non-null-assertion.md

@@ -0,0 +1,5 @@
+---
+"htmljs-parser": minor
+---
+
+Treat a trailing TypeScript non-null assertion directly after an operand (`x!`, `f()!`, `a[0]!`) as postfix, so it ends an unenclosed attribute, tag variable or statement value. `<div title=x! id="a"/>` now reports two attributes rather than one value `x! id="a"`, and `<div title=x!/>` no longer fails as an unterminated regular expression.

.changeset/type-ends-at-void.md

@@ -0,0 +1,5 @@
+---
+"htmljs-parser": minor
+---
+
+End a type at a line-final `void`. A type statement such as `export type H = () => void` carried on into the next line, and a `void` return type before `{` (`function foo(): void {`) read the body as a type; `void` inside a type is now always the `void` type rather than a prefix operator.

.changeset/valid-type-statement.md

@@ -0,0 +1,5 @@
+---
+"htmljs-parser": minor
+---
+
+`isValidStatement` now reads a statement starting with `type `, `interface ` or `declare ` as a type, as the parser does, so a multi-line type argument list is `enclosed` and a type followed by an unindented line is `invalid`.

AGENTS.md

@@ -4,7 +4,7 @@ An HTML parser recognizes content and string placeholders and allows JavaScript
 
 ## Conventions
 
-When the parser misreports a structure, fix it here rather than working around it in prettier, the compiler, or language-tools. Changing which events existing input produces breaks every consumer: treat it as a breaking change.
+When the parser misreports a structure, fix it here rather than working around it in prettier, the compiler, or language-tools. Changing the events for input that already parses and compiles correctly breaks every consumer: treat it as a breaking change. Fixing a misparse (input that errored, or compiled to the wrong code) is a feature, not a breaking change: release it as a minor, and raise the htmljs-parser floor in the compiler and prettier-plugin-marko together, since both decide what is valid with this parser.
 
 ## Agent feedback
 

src/__tests__/fixtures/attr-operators-space-after/__snapshots__/attr-operators-space-after.expected.txt

@@ -5,9 +5,10 @@
  │  │╰─ attrName
  ╰─ ╰─ tagName
 2╭─ a=x! y a
- │  │││    ╰─ attrName
- │  ││╰─ attrValue.value "x! y"
- │  │├─ attrValue "=x! y"
+ │  │││  │ ╰─ attrName
+ │  │││  ╰─ attrName
+ │  ││╰─ attrValue.value "x!"
+ │  │├─ attrValue "=x!"
  │  │╰─ attrName
  │  ├─ closeTagEnd(a)
  │  ├─ openTagEnd
@@ -368,10 +369,11 @@
   │  ├─ closeTagEnd(a)
   ╰─ ╰─ openTagStart
 48╭─ <a=x! y a/>
-  │  ││││    │╰─ openTagEnd:selfClosed "/>"
-  │  ││││    ╰─ attrName
-  │  │││╰─ attrValue.value "x! y"
-  │  ││├─ attrValue "=x! y"
+  │  ││││  │ │╰─ openTagEnd:selfClosed "/>"
+  │  ││││  │ ╰─ attrName
+  │  ││││  ╰─ attrName
+  │  │││╰─ attrValue.value "x!"
+  │  ││├─ attrValue "=x!"
   │  ││╰─ attrName
   │  │╰─ tagName
   ╰─ ╰─ openTagStart

src/__tests__/fixtures/attr-value-delete/__snapshots__/attr-value-delete.expected.txt

@@ -0,0 +1,16 @@
+1╭─ <div a=delete x.y b=delete! y c/>
+ │  ││   │││          │││         │╰─ openTagEnd:selfClosed "/>"
+ │  ││   │││          │││         ╰─ attrName
+ │  ││   │││          ││╰─ attrValue.value "delete! y"
+ │  ││   │││          │╰─ attrValue "=delete! y"
+ │  ││   │││          ╰─ attrName
+ │  ││   ││╰─ attrValue.value "delete x.y"
+ │  ││   │╰─ attrValue "=delete x.y"
+ │  ││   ╰─ attrName
+ │  │╰─ tagName "div"
+ ╰─ ╰─ openTagStart
+2╭─ static const d = delete
+ ╰─ ╰─ tagName "static"
+3├─ e.f
+4╭─ 
+ ╰─ ╰─ openTagEnd

src/__tests__/fixtures/attr-value-delete/input.marko

@@ -0,0 +1,3 @@
+<div a=delete x.y b=delete! y c/>
+static const d = delete
+e.f

src/__tests__/fixtures/attr-value-non-null-assertion-before-attr/__snapshots__/attr-value-non-null-assertion-before-attr.expected.txt

@@ -0,0 +1,11 @@
+1╭─ <div title=input.x! id="a"/>
+ │  ││   │    ││        │ ││  ╰─ openTagEnd:selfClosed "/>"
+ │  ││   │    ││        │ │╰─ attrValue.value "\"a\""
+ │  ││   │    ││        │ ╰─ attrValue "=\"a\""
+ │  ││   │    ││        ╰─ attrName "id"
+ │  ││   │    │╰─ attrValue.value "input.x!"
+ │  ││   │    ╰─ attrValue "=input.x!"
+ │  ││   ╰─ attrName "title"
+ │  │╰─ tagName "div"
+ ╰─ ╰─ openTagStart
+2╰─ 

src/__tests__/fixtures/attr-value-non-null-assertion-before-attr/input.marko

@@ -0,0 +1 @@
+<div title=input.x! id="a"/>

src/__tests__/fixtures/attr-value-non-null-assertion-before-open-tag-end/__snapshots__/attr-value-non-null-assertion-before-open-tag-end.expected.txt

@@ -0,0 +1,15 @@
+1╭─ <const/z=input.x!>
+ │  ││    ││││       ╰─ openTagEnd
+ │  ││    │││╰─ attrValue.value "input.x!"
+ │  ││    ││├─ attrValue "=input.x!"
+ │  ││    ││╰─ attrName
+ │  ││    │╰─ tagVar.value
+ │  ││    ╰─ tagVar "/z"
+ │  │╰─ tagName "const"
+ ╰─ ╰─ openTagStart
+2╭─ </const>
+ │  │ │    ╰─ closeTagEnd(const)
+ │  │ ╰─ closeTagName "const"
+ │  ├─ text "\n"
+ ╰─ ╰─ closeTagStart "</"
+3╰─ 

src/__tests__/fixtures/attr-value-non-null-assertion-before-open-tag-end/input.marko

@@ -0,0 +1,2 @@
+<const/z=input.x!>
+</const>

src/__tests__/fixtures/attr-value-non-null-assertion-before-self-close/__snapshots__/attr-value-non-null-assertion-before-self-close.expected.txt

@@ -0,0 +1,11 @@
+1╭─ <div a=a[0]! b=f()!/>
+ │  ││   │││     │││   ╰─ openTagEnd:selfClosed "/>"
+ │  ││   │││     ││╰─ attrValue.value "f()!"
+ │  ││   │││     │╰─ attrValue "=f()!"
+ │  ││   │││     ╰─ attrName
+ │  ││   ││╰─ attrValue.value "a[0]!"
+ │  ││   │╰─ attrValue "=a[0]!"
+ │  ││   ╰─ attrName
+ │  │╰─ tagName "div"
+ ╰─ ╰─ openTagStart
+2╰─ 

src/__tests__/fixtures/attr-value-non-null-assertion-before-self-close/input.marko

@@ -0,0 +1 @@
+<div a=a[0]! b=f()!/>

src/__tests__/fixtures/non-null-assertion-at-eol/__snapshots__/non-null-assertion-at-eol.expected.txt

@@ -0,0 +1,15 @@
+1╭─ static const a = b!
+ ╰─ ╰─ tagName "static"
+2╭─ div title=input.x!
+ │  │   │    │╰─ attrValue.value "input.x!"
+ │  │   │    ╰─ attrValue "=input.x!"
+ │  │   ╰─ attrName "title"
+ │  ├─ openTagEnd
+ ╰─ ╰─ tagName "div"
+3╭─   span
+ │  │ ╰─ tagName "span"
+ ╰─ ╰─ openTagEnd
+4╭─ 
+ │  ├─ openTagEnd
+ │  ├─ closeTagEnd(div)
+ ╰─ ╰─ closeTagEnd(span)

src/__tests__/fixtures/non-null-assertion-at-eol/input.marko

@@ -0,0 +1,3 @@
+static const a = b!
+div title=input.x!
+  span

src/__tests__/fixtures/placeholder-empty/__snapshots__/placeholder-empty.expected.txt

@@ -1,7 +1,5 @@
 1╭─ <div>${}</div>
- │  ││  ││ ├─ placeholder:escape.value "}</div>"
- │  ││  ││ ╰─ error(MALFORMED_PLACEHOLDER:Invalid placeholder, the expression cannot be missing)
- │  ││  │╰─ placeholder:escape "${}</div>"
+ │  ││  │  ╰─ error(MALFORMED_PLACEHOLDER:Invalid placeholder, the expression cannot be missing)
  │  ││  ╰─ openTagEnd
  │  │╰─ tagName "div"
  ╰─ ╰─ openTagStart

src/__tests__/fixtures/prefix-not-after-keyword-operator/__snapshots__/prefix-not-after-keyword-operator.expected.txt

@@ -0,0 +1,30 @@
+1╭─ static const a = typeof!
+ ╰─ ╰─ tagName "static"
+2├─ b
+3╭─ static const c = d in!
+ │  ├─ openTagEnd
+ ╰─ ╰─ tagName "static"
+4├─ e
+5╭─ static const f = g instanceof!
+ │  ├─ openTagEnd
+ ╰─ ╰─ tagName "static"
+6├─ h
+7╭─ static const i = main!
+ │  ├─ openTagEnd
+ ╰─ ╰─ tagName "static"
+8╭─ <div a=typeof! x b=void! y c=input.new! d/>
+ │  ││   │││         │││       │││          │╰─ openTagEnd:selfClosed "/>"
+ │  ││   │││         │││       │││          ╰─ attrName
+ │  ││   │││         │││       ││╰─ attrValue.value "input.new!"
+ │  ││   │││         │││       │╰─ attrValue "=input.new!"
+ │  ││   │││         │││       ╰─ attrName
+ │  ││   │││         ││╰─ attrValue.value "void! y"
+ │  ││   │││         │╰─ attrValue "=void! y"
+ │  ││   │││         ╰─ attrName
+ │  ││   ││╰─ attrValue.value "typeof! x"
+ │  ││   │╰─ attrValue "=typeof! x"
+ │  ││   ╰─ attrName
+ │  │╰─ tagName "div"
+ │  ├─ openTagEnd
+ ╰─ ╰─ openTagStart
+9╰─ 

src/__tests__/fixtures/prefix-not-after-keyword-operator/input.marko

@@ -0,0 +1,8 @@
+static const a = typeof!
+b
+static const c = d in!
+e
+static const f = g instanceof!
+h
+static const i = main!
+<div a=typeof! x b=void! y c=input.new! d/>

src/__tests__/fixtures/ts-tag-var-type-void/__snapshots__/ts-tag-var-type-void.expected.txt

@@ -0,0 +1,12 @@
+1╭─ <let/x: void>
+ │  ││  ││      ╰─ openTagEnd
+ │  ││  │╰─ tagVar.value "x: void"
+ │  ││  ╰─ tagVar "/x: void"
+ │  │╰─ tagName "let"
+ ╰─ ╰─ openTagStart
+2╭─ </let>
+ │  │ │  ╰─ closeTagEnd(let)
+ │  │ ╰─ closeTagName "let"
+ │  ├─ text "\n"
+ ╰─ ╰─ closeTagStart "</"
+3╰─ 

src/__tests__/fixtures/ts-tag-var-type-void/input.marko

@@ -0,0 +1,2 @@
+<let/x: void>
+</let>

src/__tests__/fixtures/ts-type-statement-void-before-statement/__snapshots__/ts-type-statement-void-before-statement.expected.txt

@@ -0,0 +1,7 @@
+1╭─ static type F = () => void
+ ╰─ ╰─ tagName "static"
+2╭─ static const x = 1
+ │  ├─ openTagEnd
+ ╰─ ╰─ tagName "static"
+3╭─ 
+ ╰─ ╰─ openTagEnd

src/__tests__/fixtures/ts-type-statement-void-before-statement/input.marko

@@ -0,0 +1,2 @@
+static type F = () => void
+static const x = 1

src/__tests__/fixtures/ts-type-statement-void-before-tag/__snapshots__/ts-type-statement-void-before-tag.expected.txt

@@ -0,0 +1,8 @@
+1╭─ export type H = (v: string) => void
+ ╰─ ╰─ tagName "export"
+2╭─ <div/>
+ │  ││  ╰─ openTagEnd:selfClosed "/>"
+ │  │╰─ tagName "div"
+ │  ├─ openTagEnd
+ ╰─ ╰─ openTagStart
+3╰─ 

src/__tests__/fixtures/ts-type-statement-void-before-tag/input.marko

@@ -0,0 +1,2 @@
+export type H = (v: string) => void
+<div/>

src/__tests__/fixtures/ts-void-return-type-before-body/__snapshots__/ts-void-return-type-before-body.expected.txt

@@ -0,0 +1,10 @@
+1╭─ static function foo(): void {
+ ╰─ ╰─ tagName "static"
+2├─   if (a < b) {}
+3├─ }
+4╭─ <div/>
+ │  ││  ╰─ openTagEnd:selfClosed "/>"
+ │  │╰─ tagName "div"
+ │  ├─ openTagEnd
+ ╰─ ╰─ openTagStart
+5╰─ 

src/__tests__/fixtures/ts-void-return-type-before-body/input.marko

@@ -0,0 +1,4 @@
+static function foo(): void {
+  if (a < b) {}
+}
+<div/>

src/__tests__/validate.test.ts

@@ -40,6 +40,17 @@ describe("validation helpers", () => {
     it("treats newlines in enclosed template literals as guarded", () => {
       assert.equal(isValidStatement("(`foo\nbar`)"), 2);
     });
+
+    it("reads type arguments in a type statement as a group", () => {
+      assert.equal(
+        isValidStatement("type A = Record<\n  string,\n  number\n>"),
+        2,
+      );
+    });
+
+    it("ends a type statement at an unindented newline", () => {
+      assert.equal(isValidStatement("type A = B<C>\nfoo"), 0);
+    });
   });
 
   describe("isValidScriptlet", () => {

src/states/EXPRESSION.ts

@@ -39,9 +39,13 @@ const unaryKeywords = [
   "function",
   "new",
   "typeof",
-  "void",
 ] as const;
 
+// Only JavaScript has `delete` and `void` operators; in a type `void` is the
+// `void` type. `return`, `throw` and `yield` are left out: JavaScript allows
+// no line break after them, so they never continue an expression.
+const jsUnaryKeywords = [...unaryKeywords, "delete", "void"] as const;
+
 const tsUnaryKeywords = [
   ...unaryKeywords,
   "asserts",
@@ -60,6 +64,8 @@ const binaryKeywords = [
   "satisfies",
 ] as const;
 
+const relationalKeywords = ["in", "instanceof"] as const;
+
 export const EXPRESSION: StateDefinition<ExpressionMeta> = {
   name: "EXPRESSION",
 
@@ -469,14 +475,30 @@ function lookBehindForOperator(
     case CODE.CARET:
     case CODE.COLON:
     case CODE.EQUAL:
-    case CODE.EXCLAMATION:
     case CODE.OPEN_ANGLE_BRACKET:
     case CODE.PERCENT:
     case CODE.PIPE:
     case CODE.QUESTION:
     case CODE.TILDE:
       return curPos;
 
+    case CODE.EXCLAMATION: {
+      // After an operand, `!` is a TypeScript non-null assertion (postfix);
+      // after a keyword operator (`typeof!a`, `a in!b`) it is the prefix `!`.
+      const prevCode = data.charCodeAt(curPos - 1);
+      switch (prevCode) {
+        case CODE.CLOSE_PAREN:
+        case CODE.CLOSE_SQUARE_BRACKET:
+          return -1;
+        default:
+          return isWordCode(prevCode) &&
+            lookBehindForOperator(expression, data, curPos) === -1 &&
+            lookBehindForKeyword(data, curPos - 1, relationalKeywords) === -1
+            ? -1
+            : curPos;
+      }
+    }
+
     case CODE.CLOSE_ANGLE_BRACKET:
       return data.charCodeAt(curPos - 1) === CODE.EQUAL
         ? curPos - 1
@@ -511,17 +533,11 @@ function lookBehindForOperator(
       // before `pos` is not one, no keyword can match.
       if (code < CODE.LOWER_A || code > CODE.LOWER_Z) return -1;
 
-      for (const keyword of expression.inType
-        ? tsUnaryKeywords
-        : unaryKeywords) {
-        const keywordPos = lookBehindFor(data, curPos, keyword);
-        if (keywordPos !== -1) {
-          return isWordOrPeriodCode(data.charCodeAt(keywordPos - 1))
-            ? -1
-            : keywordPos;
-        }
-      }
-      return -1;
+      return lookBehindForKeyword(
+        data,
+        curPos,
+        expression.inType ? tsUnaryKeywords : jsUnaryKeywords,
+      );
     }
   }
 }
@@ -654,6 +670,23 @@ function lookBehindWhile(
   return 0;
 }
 
+// Returns where a whole keyword ending at `pos` starts, or -1.
+function lookBehindForKeyword(
+  data: string,
+  pos: number,
+  keywords: readonly string[],
+) {
+  for (const keyword of keywords) {
+    const keywordPos = lookBehindFor(data, pos, keyword);
+    if (keywordPos !== -1) {
+      return isWordOrPeriodCode(data.charCodeAt(keywordPos - 1))
+        ? -1
+        : keywordPos;
+    }
+  }
+  return -1;
+}
+
 function lookBehindFor(data: string, pos: number, str: string) {
   let i = str.length;
   const endPos = pos - i + 1;