From d5f582dae029f558f7e23ba6c5393e8803ab7349 Mon Sep 17 00:00:00 2001 From: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:12:31 +0200 Subject: [PATCH 1/3] account for token with intermediate quotes --- normalizer_test.go | 9 ++++++++ sqllexer.go | 55 ++++++++++++++++++++++++++++++++-------------- sqllexer_test.go | 11 ++++++++++ 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/normalizer_test.go b/normalizer_test.go index 955ae8a..6c6fce9 100644 --- a/normalizer_test.go +++ b/normalizer_test.go @@ -1705,3 +1705,12 @@ func TestNormalizerDoesNotPinLargeBackingArrays(t *testing.T) { return BackingArrayTestResult{SQL: sql, Metadata: metadata}, err }) } + +func TestNormalizeQualifiedHyphenatedQuotedTable(t *testing.T) { + input := `SELECT * FROM example_schema."orders-dash"` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, input, got) + assert.Equal(t, []string{`example_schema."orders-dash"`}, metadata.Tables) +} diff --git a/sqllexer.go b/sqllexer.go index 630c3a0..aa2051b 100644 --- a/sqllexer.go +++ b/sqllexer.go @@ -396,9 +396,8 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { // If first character is Unicode, skip trie lookup if ch > 127 { - for isIdentifier(ch) { - s.hasDigits = s.hasDigits || isDigit(ch) - ch = s.nextBy(utf8.RuneLen(ch)) + if !s.consumeIdentifier() { + return s.emit(ERROR) } if s.start == s.cursor { return s.scanUnknown() @@ -449,16 +448,15 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { } // Continue scanning identifier if no keyword match - for isIdentifier(ch) { - s.hasDigits = s.hasDigits || isDigit(ch) - ch = s.nextBy(utf8.RuneLen(ch)) + if !s.consumeIdentifier() { + return s.emit(ERROR) } if s.start == s.cursor { return s.scanUnknown() } - if ch == '(' { + if s.peek() == '(' { return s.emit(FUNCTION) } if s.config.DBMS == DBMSMySQL { @@ -467,19 +465,47 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { return s.emit(IDENT) } +func (s *Lexer) consumeIdentifier() bool { + for ch := s.peek(); isIdentifier(ch); ch = s.peek() { + if s.config.DBMS != DBMSMySQL && ch == '"' && s.cursor > s.start && s.src[s.cursor-1] == '.' { // opening quote right after a `.`, ex. schema."table-name" + if !s.consumeQuotedIdentifier('"') { + return false // returns false when no closing quote is found + } + continue + } + s.hasDigits = s.hasDigits || isDigit(ch) + s.nextBy(utf8.RuneLen(ch)) + } + return true +} + func (s *Lexer) scanDoubleQuotedIdentifier(delimiter rune) *Token { return s.scanDoubleQuotedIdentifierComponent(delimiter, true) } func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualified bool) *Token { + s.start = s.cursor + s.hasQuotes = true + s.isSimpleIdentifier = true + + if !s.consumeQuotedIdentifier(delimiter) { + s.hasQuotes = false + s.isSimpleIdentifier = false + return s.emit(ERROR) + } + + if joinQualified && s.config.DBMS == DBMSMySQL { + return s.checkForSpacesInIdentifier(QUOTED_IDENT) + } + return s.emit(QUOTED_IDENT) +} + +func (s *Lexer) consumeQuotedIdentifier(delimiter rune) bool { closingDelimiter := delimiter if delimiter == '[' { closingDelimiter = ']' } - s.start = s.cursor - s.hasQuotes = true - s.isSimpleIdentifier = true firstRune := true ch := s.next() // consume the opening quote specialCase := []rune{closingDelimiter, '.', delimiter} @@ -500,9 +526,7 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie break } if isEOF(ch) { - s.hasQuotes = false // if we hit EOF, we clear the quotes - s.isSimpleIdentifier = false - return s.emit(ERROR) + return false } s.hasDigits = s.hasDigits || isDigit(ch) if s.isSimpleIdentifier { @@ -521,10 +545,7 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie ch = s.nextBy(size) } s.next() // consume the closing quote (ASCII) - if joinQualified && s.config.DBMS == DBMSMySQL { - return s.checkForSpacesInIdentifier(QUOTED_IDENT) - } - return s.emit(QUOTED_IDENT) + return true } // scanMySQLQualifiedIdentifierComponent scans an unquoted identifier after a diff --git a/sqllexer_test.go b/sqllexer_test.go index 21cc684..dcf1ac1 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1765,3 +1765,14 @@ func ExampleLexer() { fmt.Println(token) } } + +func TestLexerQualifiedHyphenatedQuotedIdentifier(t *testing.T) { + lexer := New(`example_schema."orders-dash"`) + token := lexer.Scan() + if token.Type != IDENT || token.Value != `example_schema."orders-dash"` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } +} From 96677784ee81fc842cd86f6fc091c3d435a48fcf Mon Sep 17 00:00:00 2001 From: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:10:02 +0200 Subject: [PATCH 2/3] account for mixed quotes identifier, starting with quotes --- normalizer_test.go | 20 +++++++++++++++++++- sqllexer.go | 9 +++++++++ sqllexer_test.go | 28 +++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/normalizer_test.go b/normalizer_test.go index 6c6fce9..43903f6 100644 --- a/normalizer_test.go +++ b/normalizer_test.go @@ -1706,7 +1706,7 @@ func TestNormalizerDoesNotPinLargeBackingArrays(t *testing.T) { }) } -func TestNormalizeQualifiedHyphenatedQuotedTable(t *testing.T) { +func TestNormalizeQualifiedUnquotedSchemaQuotedTable(t *testing.T) { input := `SELECT * FROM example_schema."orders-dash"` normalizer := NewNormalizer(WithCollectTables(true)) got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) @@ -1714,3 +1714,21 @@ func TestNormalizeQualifiedHyphenatedQuotedTable(t *testing.T) { assert.Equal(t, input, got) assert.Equal(t, []string{`example_schema."orders-dash"`}, metadata.Tables) } + +func TestNormalizeQuotedSchemaUnquotedTable(t *testing.T) { + input := `SELECT * FROM "example_schema".orders_underscore` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, `SELECT * FROM example_schema.orders_underscore`, got) + assert.Equal(t, []string{`example_schema.orders_underscore`}, metadata.Tables) +} + +func TestNormalizeQuotedSchemaQuotedTable(t *testing.T) { + input := `SELECT * FROM "example_schema"."orders-hyphen"` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, `SELECT * FROM example_schema.orders-hyphen`, got) + assert.Equal(t, []string{`example_schema.orders-hyphen`}, metadata.Tables) +} diff --git a/sqllexer.go b/sqllexer.go index aa2051b..89f18ce 100644 --- a/sqllexer.go +++ b/sqllexer.go @@ -497,6 +497,15 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie if joinQualified && s.config.DBMS == DBMSMySQL { return s.checkForSpacesInIdentifier(QUOTED_IDENT) } + + if joinQualified && s.peek() == '.' { // continue through an unquoted component, ex. "schema".table + s.isSimpleIdentifier = false + if !s.consumeIdentifier() { + s.hasQuotes = false + return s.emit(ERROR) + } + } + return s.emit(QUOTED_IDENT) } diff --git a/sqllexer_test.go b/sqllexer_test.go index dcf1ac1..4653009 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1766,10 +1766,32 @@ func ExampleLexer() { } } -func TestLexerQualifiedHyphenatedQuotedIdentifier(t *testing.T) { - lexer := New(`example_schema."orders-dash"`) +func TestLexerQualifiedIdentifierMixedQuoteIdentifier(t *testing.T) { + lexer := New(`example_schema."orders-hyphen"`) token := lexer.Scan() - if token.Type != IDENT || token.Value != `example_schema."orders-dash"` { + if token.Type != IDENT || token.Value != `example_schema."orders-hyphen"` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } +} + +func TestLexerQualifiedIdentifierQuotedSchemaUnquotedTable(t *testing.T) { + lexer := New(`"example_schema".orders_underscore`) // unquoted hyphen would not be part of a table name, using underscore here + token := lexer.Scan() + if token.Type != QUOTED_IDENT || token.Value != `"example_schema".orders_underscore` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } +} + +func TestLexerQualifiedIdentifierQuotedSchemaQuotedTable(t *testing.T) { + lexer := New(`"example_schema"."orders-hyphen"`) + token := lexer.Scan() + if token.Type != QUOTED_IDENT || token.Value != `"example_schema"."orders-hyphen"` { t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) } if token := lexer.Scan(); token.Type != EOF { From a3c08253bfc7e95f8deebe9108afe4dfb7f7a5d9 Mon Sep 17 00:00:00 2001 From: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:21:17 +0200 Subject: [PATCH 3/3] group tests --- normalizer_test.go | 46 +++++++++++++++++----------------- sqllexer_test.go | 61 ++++++++++++++++++++++++---------------------- 2 files changed, 56 insertions(+), 51 deletions(-) diff --git a/normalizer_test.go b/normalizer_test.go index 43903f6..9afa767 100644 --- a/normalizer_test.go +++ b/normalizer_test.go @@ -1707,28 +1707,30 @@ func TestNormalizerDoesNotPinLargeBackingArrays(t *testing.T) { } func TestNormalizeQualifiedUnquotedSchemaQuotedTable(t *testing.T) { - input := `SELECT * FROM example_schema."orders-dash"` - normalizer := NewNormalizer(WithCollectTables(true)) - got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) - assert.NoError(t, err) - assert.Equal(t, input, got) - assert.Equal(t, []string{`example_schema."orders-dash"`}, metadata.Tables) -} + t.Run("unquoted schema, quoted table", func(t *testing.T) { + input := `SELECT * FROM example_schema."orders-dash"` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, input, got) + assert.Equal(t, []string{`example_schema."orders-dash"`}, metadata.Tables) + }) -func TestNormalizeQuotedSchemaUnquotedTable(t *testing.T) { - input := `SELECT * FROM "example_schema".orders_underscore` - normalizer := NewNormalizer(WithCollectTables(true)) - got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) - assert.NoError(t, err) - assert.Equal(t, `SELECT * FROM example_schema.orders_underscore`, got) - assert.Equal(t, []string{`example_schema.orders_underscore`}, metadata.Tables) -} + t.Run("quoted schema, unquoted table", func(t *testing.T) { + input := `SELECT * FROM "example_schema".orders_underscore` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, `SELECT * FROM example_schema.orders_underscore`, got) + assert.Equal(t, []string{`example_schema.orders_underscore`}, metadata.Tables) + }) -func TestNormalizeQuotedSchemaQuotedTable(t *testing.T) { - input := `SELECT * FROM "example_schema"."orders-hyphen"` - normalizer := NewNormalizer(WithCollectTables(true)) - got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) - assert.NoError(t, err) - assert.Equal(t, `SELECT * FROM example_schema.orders-hyphen`, got) - assert.Equal(t, []string{`example_schema.orders-hyphen`}, metadata.Tables) + t.Run("quoted schema, quoted table", func(t *testing.T) { + input := `SELECT * FROM "example_schema"."orders-hyphen"` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, `SELECT * FROM example_schema.orders-hyphen`, got) + assert.Equal(t, []string{`example_schema.orders-hyphen`}, metadata.Tables) + }) } diff --git a/sqllexer_test.go b/sqllexer_test.go index 4653009..2cbc7db 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1766,35 +1766,38 @@ func ExampleLexer() { } } -func TestLexerQualifiedIdentifierMixedQuoteIdentifier(t *testing.T) { - lexer := New(`example_schema."orders-hyphen"`) - token := lexer.Scan() - if token.Type != IDENT || token.Value != `example_schema."orders-hyphen"` { - t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) - } - if token := lexer.Scan(); token.Type != EOF { - t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) - } -} +func TestLexerMixedQuoteQualifiedIdentifier(t *testing.T) { + t.Run("unquoted schema, quoted table", func(t *testing.T) { + lexer := New(`example_schema."orders-hyphen"`) + token := lexer.Scan() + if token.Type != IDENT || token.Value != `example_schema."orders-hyphen"` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } + }) -func TestLexerQualifiedIdentifierQuotedSchemaUnquotedTable(t *testing.T) { - lexer := New(`"example_schema".orders_underscore`) // unquoted hyphen would not be part of a table name, using underscore here - token := lexer.Scan() - if token.Type != QUOTED_IDENT || token.Value != `"example_schema".orders_underscore` { - t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) - } - if token := lexer.Scan(); token.Type != EOF { - t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) - } -} + t.Run("quoted schema, unquoted table", func(t *testing.T) { + lexer := New(`"example_schema".orders_underscore`) // unquoted hyphen would not be part of a table name, using underscore here + token := lexer.Scan() + if token.Type != QUOTED_IDENT || token.Value != `"example_schema".orders_underscore` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } + }) + + t.Run("quoted schema, quoted table", func(t *testing.T) { + lexer := New(`"example_schema"."orders-hyphen"`) + token := lexer.Scan() + if token.Type != QUOTED_IDENT || token.Value != `"example_schema"."orders-hyphen"` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } + }) -func TestLexerQualifiedIdentifierQuotedSchemaQuotedTable(t *testing.T) { - lexer := New(`"example_schema"."orders-hyphen"`) - token := lexer.Scan() - if token.Type != QUOTED_IDENT || token.Value != `"example_schema"."orders-hyphen"` { - t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) - } - if token := lexer.Scan(); token.Type != EOF { - t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) - } }