From f813928fc37f06ee5a6182a3e1c369d88bfc2314 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 16 Oct 2026 12:35:29 -0400 Subject: [PATCH] Replace TokenType and TokenizerState enums with literal union types BREAKING CHANGE: `TokenType` and `TokenizerState` are now type-only exports (string literal unions) rather than TypeScript enums, so they no longer exist at runtime. The underlying values are unchanged: tokens still have `type: 'StartTag'` etc., and tokenizer states are still the same camelCase strings, so code that compares against string literals is unaffected. Code that used the enums as values must switch to string literals: // before if (token.type === TokenType.StartTag) { ... } tokenizer.transitionTo(TokenizerState.data); // after if (token.type === 'StartTag') { ... } tokenizer.transitionTo('data'); Importing `TokenType` or `TokenizerState` for use in type positions continues to work. Consumers compiling with `isolatedModules` or `verbatimModuleSyntax` should re-export them with `export type`. The tokenizer-states codegen now emits a union type as well. Co-Authored-By: Claude Opus 5.5 (1M context) --- commands/codegen/tokenizer-states.js | 5 +- src/evented-tokenizer.ts | 166 +++++++++++++-------------- src/generated/tokenizer-states.ts | 165 +++++++++++++------------- src/tokenizer.ts | 26 ++--- src/types.ts | 20 ++-- tests/tokenizer-tests.ts | 11 +- 6 files changed, 192 insertions(+), 201 deletions(-) diff --git a/commands/codegen/tokenizer-states.js b/commands/codegen/tokenizer-states.js index aaf854b..e4f0bc4 100644 --- a/commands/codegen/tokenizer-states.js +++ b/commands/codegen/tokenizer-states.js @@ -10,8 +10,7 @@ export default function buildTokenizerStates(syntax) { }); states.push(state); } - return `export enum TokenizerState { -${states.map(s => ` ${s} = '${s}'`).join(',\n')} -} + return `export type TokenizerState = +${states.map(s => ` | '${s}'`).join('\n')}; `; } diff --git a/src/evented-tokenizer.ts b/src/evented-tokenizer.ts index 1ed23b3..096f4ba 100644 --- a/src/evented-tokenizer.ts +++ b/src/evented-tokenizer.ts @@ -2,7 +2,7 @@ import { preprocessInput, isSpace } from './utils.js'; import { EntityParser, TokenizerDelegate, TokenizerState } from './types.js'; export default class EventedTokenizer { - public state: TokenizerState = TokenizerState.beforeData; + public state: TokenizerState = 'beforeData'; public line = -1; public column = -1; @@ -21,7 +21,7 @@ export default class EventedTokenizer { } reset() { - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); this.input = ''; this.tagNameBuffer = ''; @@ -62,7 +62,7 @@ export default class EventedTokenizer { flushData() { if (this.state === 'data') { this.delegate.finishData(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } } @@ -128,7 +128,7 @@ export default class EventedTokenizer { let char = this.peek(); if (char === '<' && !this.isIgnoredEndTag()) { - this.transitionTo(TokenizerState.tagOpen); + this.transitionTo('tagOpen'); this.markTagStart(); this.consume(); } else { @@ -140,7 +140,7 @@ export default class EventedTokenizer { } } - this.transitionTo(TokenizerState.data); + this.transitionTo('data'); this.delegate.beginData(); } }, @@ -151,7 +151,7 @@ export default class EventedTokenizer { if (char === '<' && !this.isIgnoredEndTag()) { this.delegate.finishData(); - this.transitionTo(TokenizerState.tagOpen); + this.transitionTo('tagOpen'); this.markTagStart(); this.consume(); } else if (char === '&' && tag !== 'script' && tag !== 'style') { @@ -167,11 +167,11 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '!') { - this.transitionTo(TokenizerState.markupDeclarationOpen); + this.transitionTo('markupDeclarationOpen'); } else if (char === '/') { - this.transitionTo(TokenizerState.endTagOpen); + this.transitionTo('endTagOpen'); } else if (!isSpace(char)) { - this.transitionTo(TokenizerState.tagName); + this.transitionTo('tagName'); this.tagNameBuffer = ''; this.delegate.beginStartTag(); this.appendToTagName(char); @@ -183,7 +183,7 @@ export default class EventedTokenizer { if (char === '-' && this.peek() === '-') { this.consume(); - this.transitionTo(TokenizerState.commentStart); + this.transitionTo('commentStart'); this.delegate.beginComment(); } else { let maybeDoctype = char.toUpperCase() + this.input.substring(this.index, this.index + 6).toUpperCase(); @@ -195,7 +195,7 @@ export default class EventedTokenizer { this.consume(); this.consume(); this.consume(); - this.transitionTo(TokenizerState.doctype); + this.transitionTo('doctype'); if (this.delegate.beginDoctype) this.delegate.beginDoctype(); } } @@ -205,7 +205,7 @@ export default class EventedTokenizer { let char = this.consume(); if (isSpace(char)) { - this.transitionTo(TokenizerState.beforeDoctypeName); + this.transitionTo('beforeDoctypeName'); } }, @@ -215,7 +215,7 @@ export default class EventedTokenizer { if (isSpace(char)) { return; } else { - this.transitionTo(TokenizerState.doctypeName); + this.transitionTo('doctypeName'); if (this.delegate.appendToDoctypeName) this.delegate.appendToDoctypeName(char.toLowerCase()); } }, @@ -224,10 +224,10 @@ export default class EventedTokenizer { let char = this.consume(); if (isSpace(char)) { - this.transitionTo(TokenizerState.afterDoctypeName); + this.transitionTo('afterDoctypeName'); } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { if (this.delegate.appendToDoctypeName) this.delegate.appendToDoctypeName(char.toLowerCase()); } @@ -240,7 +240,7 @@ export default class EventedTokenizer { return; } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { let nextSixChars = char.toUpperCase() + this.input.substring(this.index, this.index + 5).toUpperCase(); @@ -257,9 +257,9 @@ export default class EventedTokenizer { } if (isPublic) { - this.transitionTo(TokenizerState.afterDoctypePublicKeyword); + this.transitionTo('afterDoctypePublicKeyword'); } else if (isSystem) { - this.transitionTo(TokenizerState.afterDoctypeSystemKeyword); + this.transitionTo('afterDoctypeSystemKeyword'); } } }, @@ -268,18 +268,18 @@ export default class EventedTokenizer { let char = this.peek(); if (isSpace(char)) { - this.transitionTo(TokenizerState.beforeDoctypePublicIdentifier); + this.transitionTo('beforeDoctypePublicIdentifier'); this.consume(); } else if (char === '"') { - this.transitionTo(TokenizerState.doctypePublicIdentifierDoubleQuoted); + this.transitionTo('doctypePublicIdentifierDoubleQuoted'); this.consume(); } else if (char === "'") { - this.transitionTo(TokenizerState.doctypePublicIdentifierSingleQuoted); + this.transitionTo('doctypePublicIdentifierSingleQuoted'); this.consume(); } else if (char === '>') { this.consume(); if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } }, @@ -287,10 +287,10 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '"') { - this.transitionTo(TokenizerState.afterDoctypePublicIdentifier); + this.transitionTo('afterDoctypePublicIdentifier'); } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { if (this.delegate.appendToDoctypePublicIdentifier) this.delegate.appendToDoctypePublicIdentifier(char); } @@ -300,10 +300,10 @@ export default class EventedTokenizer { let char = this.consume(); if (char === "'") { - this.transitionTo(TokenizerState.afterDoctypePublicIdentifier); + this.transitionTo('afterDoctypePublicIdentifier'); } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { if (this.delegate.appendToDoctypePublicIdentifier) this.delegate.appendToDoctypePublicIdentifier(char); } @@ -313,14 +313,14 @@ export default class EventedTokenizer { let char = this.consume(); if (isSpace(char)) { - this.transitionTo(TokenizerState.betweenDoctypePublicAndSystemIdentifiers); + this.transitionTo('betweenDoctypePublicAndSystemIdentifiers'); } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else if (char === '"') { - this.transitionTo(TokenizerState.doctypeSystemIdentifierDoubleQuoted); + this.transitionTo('doctypeSystemIdentifierDoubleQuoted'); } else if (char === "'") { - this.transitionTo(TokenizerState.doctypeSystemIdentifierSingleQuoted); + this.transitionTo('doctypeSystemIdentifierSingleQuoted'); } }, @@ -331,11 +331,11 @@ export default class EventedTokenizer { return; } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else if (char === '"') { - this.transitionTo(TokenizerState.doctypeSystemIdentifierDoubleQuoted); + this.transitionTo('doctypeSystemIdentifierDoubleQuoted'); } else if (char === "'") { - this.transitionTo(TokenizerState.doctypeSystemIdentifierSingleQuoted); + this.transitionTo('doctypeSystemIdentifierSingleQuoted'); } }, @@ -343,10 +343,10 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '"') { - this.transitionTo(TokenizerState.afterDoctypeSystemIdentifier); + this.transitionTo('afterDoctypeSystemIdentifier'); } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { if (this.delegate.appendToDoctypeSystemIdentifier) this.delegate.appendToDoctypeSystemIdentifier(char); } @@ -356,10 +356,10 @@ export default class EventedTokenizer { let char = this.consume(); if (char === "'") { - this.transitionTo(TokenizerState.afterDoctypeSystemIdentifier); + this.transitionTo('afterDoctypeSystemIdentifier'); } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { if (this.delegate.appendToDoctypeSystemIdentifier) this.delegate.appendToDoctypeSystemIdentifier(char); } @@ -372,7 +372,7 @@ export default class EventedTokenizer { return; } else if (char === '>') { if (this.delegate.endDoctype) this.delegate.endDoctype(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } }, @@ -380,13 +380,13 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '-') { - this.transitionTo(TokenizerState.commentStartDash); + this.transitionTo('commentStartDash'); } else if (char === '>') { this.delegate.finishComment(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { this.delegate.appendToCommentData(char); - this.transitionTo(TokenizerState.comment); + this.transitionTo('comment'); } }, @@ -394,13 +394,13 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '-') { - this.transitionTo(TokenizerState.commentEnd); + this.transitionTo('commentEnd'); } else if (char === '>') { this.delegate.finishComment(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { this.delegate.appendToCommentData('-'); - this.transitionTo(TokenizerState.comment); + this.transitionTo('comment'); } }, @@ -408,7 +408,7 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '-') { - this.transitionTo(TokenizerState.commentEndDash); + this.transitionTo('commentEndDash'); } else { this.delegate.appendToCommentData(char); } @@ -418,10 +418,10 @@ export default class EventedTokenizer { let char = this.consume(); if (char === '-') { - this.transitionTo(TokenizerState.commentEnd); + this.transitionTo('commentEnd'); } else { this.delegate.appendToCommentData('-' + char); - this.transitionTo(TokenizerState.comment); + this.transitionTo('comment'); } }, @@ -430,10 +430,10 @@ export default class EventedTokenizer { if (char === '>') { this.delegate.finishComment(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { this.delegate.appendToCommentData('--' + char); - this.transitionTo(TokenizerState.comment); + this.transitionTo('comment'); } }, @@ -441,12 +441,12 @@ export default class EventedTokenizer { let char = this.consume(); if (isSpace(char)) { - this.transitionTo(TokenizerState.beforeAttributeName); + this.transitionTo('beforeAttributeName'); } else if (char === '/') { - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); } else if (char === '>') { this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { this.appendToTagName(char); } @@ -456,14 +456,14 @@ export default class EventedTokenizer { let char = this.consume(); if (isSpace(char)) { - this.transitionTo(TokenizerState.beforeAttributeName); + this.transitionTo('beforeAttributeName'); this.tagNameBuffer = ''; } else if (char === '/') { - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); this.tagNameBuffer = ''; } else if (char === '>') { this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); this.tagNameBuffer = ''; } else { this.appendToTagName(char); @@ -477,22 +477,22 @@ export default class EventedTokenizer { this.consume(); return; } else if (char === '/') { - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); this.consume(); } else if (char === '>') { this.consume(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else if (char === '=') { this.delegate.reportSyntaxError( 'attribute name cannot start with equals sign' ); - this.transitionTo(TokenizerState.attributeName); + this.transitionTo('attributeName'); this.delegate.beginAttribute(); this.consume(); this.delegate.appendToAttributeName(char); } else { - this.transitionTo(TokenizerState.attributeName); + this.transitionTo('attributeName'); this.delegate.beginAttribute(); } }, @@ -501,22 +501,22 @@ export default class EventedTokenizer { let char = this.peek(); if (isSpace(char)) { - this.transitionTo(TokenizerState.afterAttributeName); + this.transitionTo('afterAttributeName'); this.consume(); } else if (char === '/') { this.delegate.beginAttributeValue(false); this.delegate.finishAttributeValue(); this.consume(); - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); } else if (char === '=') { - this.transitionTo(TokenizerState.beforeAttributeValue); + this.transitionTo('beforeAttributeValue'); this.consume(); } else if (char === '>') { this.delegate.beginAttributeValue(false); this.delegate.finishAttributeValue(); this.consume(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else if (char === '"' || char === "'" || char === '<') { this.delegate.reportSyntaxError( char + ' is not a valid character within attribute names' @@ -539,20 +539,20 @@ export default class EventedTokenizer { this.delegate.beginAttributeValue(false); this.delegate.finishAttributeValue(); this.consume(); - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); } else if (char === '=') { this.consume(); - this.transitionTo(TokenizerState.beforeAttributeValue); + this.transitionTo('beforeAttributeValue'); } else if (char === '>') { this.delegate.beginAttributeValue(false); this.delegate.finishAttributeValue(); this.consume(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { this.delegate.beginAttributeValue(false); this.delegate.finishAttributeValue(); - this.transitionTo(TokenizerState.attributeName); + this.transitionTo('attributeName'); this.delegate.beginAttribute(); this.consume(); this.delegate.appendToAttributeName(char); @@ -565,11 +565,11 @@ export default class EventedTokenizer { if (isSpace(char)) { this.consume(); } else if (char === '"') { - this.transitionTo(TokenizerState.attributeValueDoubleQuoted); + this.transitionTo('attributeValueDoubleQuoted'); this.delegate.beginAttributeValue(true); this.consume(); } else if (char === "'") { - this.transitionTo(TokenizerState.attributeValueSingleQuoted); + this.transitionTo('attributeValueSingleQuoted'); this.delegate.beginAttributeValue(true); this.consume(); } else if (char === '>') { @@ -577,9 +577,9 @@ export default class EventedTokenizer { this.delegate.finishAttributeValue(); this.consume(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { - this.transitionTo(TokenizerState.attributeValueUnquoted); + this.transitionTo('attributeValueUnquoted'); this.delegate.beginAttributeValue(false); this.consume(); this.delegate.appendToAttributeValue(char); @@ -591,7 +591,7 @@ export default class EventedTokenizer { if (char === '"') { this.delegate.finishAttributeValue(); - this.transitionTo(TokenizerState.afterAttributeValueQuoted); + this.transitionTo('afterAttributeValueQuoted'); } else if (char === '&') { this.delegate.appendToAttributeValue(this.consumeCharRef() || '&'); } else { @@ -604,7 +604,7 @@ export default class EventedTokenizer { if (char === "'") { this.delegate.finishAttributeValue(); - this.transitionTo(TokenizerState.afterAttributeValueQuoted); + this.transitionTo('afterAttributeValueQuoted'); } else if (char === '&') { this.delegate.appendToAttributeValue(this.consumeCharRef() || '&'); } else { @@ -618,11 +618,11 @@ export default class EventedTokenizer { if (isSpace(char)) { this.delegate.finishAttributeValue(); this.consume(); - this.transitionTo(TokenizerState.beforeAttributeName); + this.transitionTo('beforeAttributeName'); } else if (char === '/') { this.delegate.finishAttributeValue(); this.consume(); - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); } else if (char === '&') { this.consume(); this.delegate.appendToAttributeValue(this.consumeCharRef() || '&'); @@ -630,7 +630,7 @@ export default class EventedTokenizer { this.delegate.finishAttributeValue(); this.consume(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { this.consume(); this.delegate.appendToAttributeValue(char); @@ -642,16 +642,16 @@ export default class EventedTokenizer { if (isSpace(char)) { this.consume(); - this.transitionTo(TokenizerState.beforeAttributeName); + this.transitionTo('beforeAttributeName'); } else if (char === '/') { this.consume(); - this.transitionTo(TokenizerState.selfClosingStartTag); + this.transitionTo('selfClosingStartTag'); } else if (char === '>') { this.consume(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { - this.transitionTo(TokenizerState.beforeAttributeName); + this.transitionTo('beforeAttributeName'); } }, @@ -662,9 +662,9 @@ export default class EventedTokenizer { this.consume(); this.delegate.markTagAsSelfClosing(); this.delegate.finishTag(); - this.transitionTo(TokenizerState.beforeData); + this.transitionTo('beforeData'); } else { - this.transitionTo(TokenizerState.beforeAttributeName); + this.transitionTo('beforeAttributeName'); } }, @@ -672,7 +672,7 @@ export default class EventedTokenizer { let char = this.consume(); if (!isSpace(char)) { - this.transitionTo(TokenizerState.endTagName); + this.transitionTo('endTagName'); this.tagNameBuffer = ''; this.delegate.beginEndTag(); this.appendToTagName(char); diff --git a/src/generated/tokenizer-states.ts b/src/generated/tokenizer-states.ts index eea4f5a..f3f86dc 100644 --- a/src/generated/tokenizer-states.ts +++ b/src/generated/tokenizer-states.ts @@ -2,86 +2,85 @@ * generated from https://raw.githubusercontent.com/w3c/html/26b5126f96f736f796b9e29718138919dd513744/sections/syntax.include * do not edit */ -export enum TokenizerState { - beforeData = 'beforeData', - data = 'data', - rcdata = 'rcdata', - rawtext = 'rawtext', - scriptData = 'scriptData', - plaintext = 'plaintext', - tagOpen = 'tagOpen', - endTagOpen = 'endTagOpen', - tagName = 'tagName', - endTagName = 'endTagName', - rcdataLessThanSign = 'rcdataLessThanSign', - rcdataEndTagOpen = 'rcdataEndTagOpen', - rcdataEndTagName = 'rcdataEndTagName', - rawtextLessThanSign = 'rawtextLessThanSign', - rawtextEndTagOpen = 'rawtextEndTagOpen', - rawtextEndTagName = 'rawtextEndTagName', - scriptDataLessThanSign = 'scriptDataLessThanSign', - scriptDataEndTagOpen = 'scriptDataEndTagOpen', - scriptDataEndTagName = 'scriptDataEndTagName', - scriptDataEscapeStart = 'scriptDataEscapeStart', - scriptDataEscapseStartDash = 'scriptDataEscapseStartDash', - scriptDataEscaped = 'scriptDataEscaped', - scriptDataEscapedDash = 'scriptDataEscapedDash', - scriptDataEscapedDashDash = 'scriptDataEscapedDashDash', - scriptDataEscapedLessThanSign = 'scriptDataEscapedLessThanSign', - scriptDataEscapedEndTagOpen = 'scriptDataEscapedEndTagOpen', - scriptDataEscapedEndTagName = 'scriptDataEscapedEndTagName', - scriptDataDoubleEscapeStart = 'scriptDataDoubleEscapeStart', - scriptDataDoubleEscaped = 'scriptDataDoubleEscaped', - scriptDataDoubleEscapedDash = 'scriptDataDoubleEscapedDash', - scriptDataDoubleEscapedDashDash = 'scriptDataDoubleEscapedDashDash', - scriptDataDoubleEscapedLessThanSign = 'scriptDataDoubleEscapedLessThanSign', - scriptDataDoubleEscapeEnd = 'scriptDataDoubleEscapeEnd', - beforeAttributeName = 'beforeAttributeName', - attributeName = 'attributeName', - afterAttributeName = 'afterAttributeName', - beforeAttributeValue = 'beforeAttributeValue', - attributeValueDoubleQuoted = 'attributeValueDoubleQuoted', - attributeValueSingleQuoted = 'attributeValueSingleQuoted', - attributeValueUnquoted = 'attributeValueUnquoted', - afterAttributeValueQuoted = 'afterAttributeValueQuoted', - selfClosingStartTag = 'selfClosingStartTag', - bogusComment = 'bogusComment', - markupDeclarationOpen = 'markupDeclarationOpen', - commentStart = 'commentStart', - commentStartDash = 'commentStartDash', - comment = 'comment', - commentLessThanSign = 'commentLessThanSign', - commentLessThanSignBang = 'commentLessThanSignBang', - commentLessThanSignBangDash = 'commentLessThanSignBangDash', - commentLessThanSignBangDashDash = 'commentLessThanSignBangDashDash', - commentEndDash = 'commentEndDash', - commentEnd = 'commentEnd', - commentEndBang = 'commentEndBang', - doctype = 'doctype', - beforeDoctypeName = 'beforeDoctypeName', - doctypeName = 'doctypeName', - afterDoctypeName = 'afterDoctypeName', - afterDoctypePublicKeyword = 'afterDoctypePublicKeyword', - beforeDoctypePublicIdentifier = 'beforeDoctypePublicIdentifier', - doctypePublicIdentifierDoubleQuoted = 'doctypePublicIdentifierDoubleQuoted', - doctypePublicIdentifierSingleQuoted = 'doctypePublicIdentifierSingleQuoted', - afterDoctypePublicIdentifier = 'afterDoctypePublicIdentifier', - betweenDoctypePublicAndSystemIdentifiers = 'betweenDoctypePublicAndSystemIdentifiers', - afterDoctypeSystemKeyword = 'afterDoctypeSystemKeyword', - beforeDoctypeSystemIdentifier = 'beforeDoctypeSystemIdentifier', - doctypeSystemIdentifierDoubleQuoted = 'doctypeSystemIdentifierDoubleQuoted', - doctypeSystemIdentifierSingleQuoted = 'doctypeSystemIdentifierSingleQuoted', - afterDoctypeSystemIdentifier = 'afterDoctypeSystemIdentifier', - bogusDoctype = 'bogusDoctype', - cdataSection = 'cdataSection', - cdataSectionBracket = 'cdataSectionBracket', - cdataSectionEnd = 'cdataSectionEnd', - characterReference = 'characterReference', - numericCharacterReference = 'numericCharacterReference', - hexadecimalCharacterReferenceStart = 'hexadecimalCharacterReferenceStart', - decimalCharacterReferenceStart = 'decimalCharacterReferenceStart', - hexadecimalCharacterReference = 'hexadecimalCharacterReference', - decimalCharacterReference = 'decimalCharacterReference', - numericCharacterReferenceEnd = 'numericCharacterReferenceEnd', - characterReferenceEnd = 'characterReferenceEnd' -} +export type TokenizerState = + | 'beforeData' + | 'data' + | 'rcdata' + | 'rawtext' + | 'scriptData' + | 'plaintext' + | 'tagOpen' + | 'endTagOpen' + | 'tagName' + | 'endTagName' + | 'rcdataLessThanSign' + | 'rcdataEndTagOpen' + | 'rcdataEndTagName' + | 'rawtextLessThanSign' + | 'rawtextEndTagOpen' + | 'rawtextEndTagName' + | 'scriptDataLessThanSign' + | 'scriptDataEndTagOpen' + | 'scriptDataEndTagName' + | 'scriptDataEscapeStart' + | 'scriptDataEscapseStartDash' + | 'scriptDataEscaped' + | 'scriptDataEscapedDash' + | 'scriptDataEscapedDashDash' + | 'scriptDataEscapedLessThanSign' + | 'scriptDataEscapedEndTagOpen' + | 'scriptDataEscapedEndTagName' + | 'scriptDataDoubleEscapeStart' + | 'scriptDataDoubleEscaped' + | 'scriptDataDoubleEscapedDash' + | 'scriptDataDoubleEscapedDashDash' + | 'scriptDataDoubleEscapedLessThanSign' + | 'scriptDataDoubleEscapeEnd' + | 'beforeAttributeName' + | 'attributeName' + | 'afterAttributeName' + | 'beforeAttributeValue' + | 'attributeValueDoubleQuoted' + | 'attributeValueSingleQuoted' + | 'attributeValueUnquoted' + | 'afterAttributeValueQuoted' + | 'selfClosingStartTag' + | 'bogusComment' + | 'markupDeclarationOpen' + | 'commentStart' + | 'commentStartDash' + | 'comment' + | 'commentLessThanSign' + | 'commentLessThanSignBang' + | 'commentLessThanSignBangDash' + | 'commentLessThanSignBangDashDash' + | 'commentEndDash' + | 'commentEnd' + | 'commentEndBang' + | 'doctype' + | 'beforeDoctypeName' + | 'doctypeName' + | 'afterDoctypeName' + | 'afterDoctypePublicKeyword' + | 'beforeDoctypePublicIdentifier' + | 'doctypePublicIdentifierDoubleQuoted' + | 'doctypePublicIdentifierSingleQuoted' + | 'afterDoctypePublicIdentifier' + | 'betweenDoctypePublicAndSystemIdentifiers' + | 'afterDoctypeSystemKeyword' + | 'beforeDoctypeSystemIdentifier' + | 'doctypeSystemIdentifierDoubleQuoted' + | 'doctypeSystemIdentifierSingleQuoted' + | 'afterDoctypeSystemIdentifier' + | 'bogusDoctype' + | 'cdataSection' + | 'cdataSectionBracket' + | 'cdataSectionEnd' + | 'characterReference' + | 'numericCharacterReference' + | 'hexadecimalCharacterReferenceStart' + | 'decimalCharacterReferenceStart' + | 'hexadecimalCharacterReference' + | 'decimalCharacterReference' + | 'numericCharacterReferenceEnd' + | 'characterReferenceEnd'; diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 29ef6c0..71639cc 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -101,17 +101,17 @@ export default class Tokenizer implements TokenizerDelegate { beginDoctype() { this.push({ - type: TokenType.Doctype, + type: 'Doctype', name: '', }); } appendToDoctypeName(char: string) { - this.current(TokenType.Doctype).name += char; + this.current('Doctype').name += char; } appendToDoctypePublicIdentifier(char: string) { - let doctype = this.current(TokenType.Doctype); + let doctype = this.current('Doctype'); if (doctype.publicIdentifier === undefined) { doctype.publicIdentifier = char; @@ -121,7 +121,7 @@ export default class Tokenizer implements TokenizerDelegate { } appendToDoctypeSystemIdentifier(char: string) { - let doctype = this.current(TokenType.Doctype); + let doctype = this.current('Doctype'); if (doctype.systemIdentifier === undefined) { doctype.systemIdentifier = char; @@ -136,13 +136,13 @@ export default class Tokenizer implements TokenizerDelegate { beginData() { this.push({ - type: TokenType.Chars, + type: 'Chars', chars: '' }); } appendToData(char: string) { - this.current(TokenType.Chars).chars += char; + this.current('Chars').chars += char; } finishData() { @@ -153,13 +153,13 @@ export default class Tokenizer implements TokenizerDelegate { beginComment() { this.push({ - type: TokenType.Comment, + type: 'Comment', chars: '' }); } appendToCommentData(char: string) { - this.current(TokenType.Comment).chars += char; + this.current('Comment').chars += char; } finishComment() { @@ -172,7 +172,7 @@ export default class Tokenizer implements TokenizerDelegate { beginStartTag() { this.push({ - type: TokenType.StartTag, + type: 'StartTag', tagName: '', attributes: [], selfClosing: false @@ -181,7 +181,7 @@ export default class Tokenizer implements TokenizerDelegate { beginEndTag() { this.push({ - type: TokenType.EndTag, + type: 'EndTag', tagName: '' }); } @@ -191,12 +191,12 @@ export default class Tokenizer implements TokenizerDelegate { } markTagAsSelfClosing() { - this.current(TokenType.StartTag).selfClosing = true; + this.current('StartTag').selfClosing = true; } // Tags - name appendToTagName(char: string) { - this.current(TokenType.StartTag, TokenType.EndTag).tagName += char; + this.current('StartTag', 'EndTag').tagName += char; } // Tags - attributes @@ -218,7 +218,7 @@ export default class Tokenizer implements TokenizerDelegate { } finishAttributeValue() { - this.current(TokenType.StartTag).attributes.push(this._currentAttribute!); + this.current('StartTag').attributes.push(this._currentAttribute!); } reportSyntaxError(message: string) { diff --git a/src/types.ts b/src/types.ts index bb230be..69f640f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -30,39 +30,33 @@ export interface TokenBase { loc?: Location; } -export interface Doctype extends TokenBase { +export interface Doctype extends TokenBase<'Doctype'> { name: string; publicIdentifier?: string; systemIdentifier?: string; } -export interface StartTag extends TokenBase { +export interface StartTag extends TokenBase<'StartTag'> { tagName: string; attributes: Attribute[]; selfClosing: boolean; } -export interface EndTag extends TokenBase { +export interface EndTag extends TokenBase<'EndTag'> { tagName: string; } -export interface Chars extends TokenBase { +export interface Chars extends TokenBase<'Chars'> { chars: string; } -export interface Comment extends TokenBase { +export interface Comment extends TokenBase<'Comment'> { chars: string; } export type Token = StartTag | EndTag | Chars | Comment | Doctype; -export enum TokenType { - Doctype = 'Doctype', - StartTag = 'StartTag', - EndTag = 'EndTag', - Chars = 'Chars', - Comment = 'Comment' -} +export type TokenType = 'Doctype' | 'StartTag' | 'EndTag' | 'Chars' | 'Comment'; export interface TokenMap { StartTag: StartTag; @@ -108,4 +102,4 @@ export interface TokenizerDelegate { reportSyntaxError(error: string): void; } -export { TokenizerState } from './generated/tokenizer-states.js'; +export type { TokenizerState } from './generated/tokenizer-states.js'; diff --git a/tests/tokenizer-tests.ts b/tests/tokenizer-tests.ts index 7fe9240..7bec388 100644 --- a/tests/tokenizer-tests.ts +++ b/tests/tokenizer-tests.ts @@ -9,7 +9,6 @@ import { Comment, Chars, Token, - TokenType, Attribute } from 'simple-html-tokenizer'; @@ -619,14 +618,14 @@ QUnit.test('tokens: comment start-tag Chars end-tag', function(assert) { function chars(s?: string): Chars { return { - type: TokenType.Chars, + type: 'Chars', chars: s === undefined ? '' : s }; } function comment(s?: string): Comment { return { - type: TokenType.Comment, + type: 'Comment', chars: s === undefined ? '' : s }; } @@ -637,7 +636,7 @@ function startTag( selfClosing?: boolean ): StartTag { return { - type: TokenType.StartTag, + type: 'StartTag', tagName: tagName, attributes: attributes === undefined ? [] : attributes, selfClosing: selfClosing === undefined ? false : selfClosing @@ -646,14 +645,14 @@ function startTag( function endTag(tagName: string): EndTag { return { - type: TokenType.EndTag, + type: 'EndTag', tagName: tagName }; } function doctype(publicIdentifier?: string, systemIdentifier?: string): Doctype { let doctype: Doctype = { - type: TokenType.Doctype, + type: 'Doctype', name: 'html', };