diff --git a/normalizer_test.go b/normalizer_test.go index 955ae8a..9afa767 100644 --- a/normalizer_test.go +++ b/normalizer_test.go @@ -1705,3 +1705,32 @@ func TestNormalizerDoesNotPinLargeBackingArrays(t *testing.T) { return BackingArrayTestResult{SQL: sql, Metadata: metadata}, err }) } + +func TestNormalizeQualifiedUnquotedSchemaQuotedTable(t *testing.T) { + t.Run("unquoted schema, quoted table", func(t *testing.T) { + input := `SELECT * FROM example_schema."orders-dash"` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, input, got) + assert.Equal(t, []string{`example_schema."orders-dash"`}, metadata.Tables) + }) + + t.Run("quoted schema, unquoted table", func(t *testing.T) { + input := `SELECT * FROM "example_schema".orders_underscore` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, `SELECT * FROM example_schema.orders_underscore`, got) + assert.Equal(t, []string{`example_schema.orders_underscore`}, metadata.Tables) + }) + + t.Run("quoted schema, quoted table", func(t *testing.T) { + input := `SELECT * FROM "example_schema"."orders-hyphen"` + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres)) + assert.NoError(t, err) + assert.Equal(t, `SELECT * FROM example_schema.orders-hyphen`, got) + assert.Equal(t, []string{`example_schema.orders-hyphen`}, metadata.Tables) + }) +} diff --git a/sqllexer.go b/sqllexer.go index 630c3a0..89f18ce 100644 --- a/sqllexer.go +++ b/sqllexer.go @@ -396,9 +396,8 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { // If first character is Unicode, skip trie lookup if ch > 127 { - for isIdentifier(ch) { - s.hasDigits = s.hasDigits || isDigit(ch) - ch = s.nextBy(utf8.RuneLen(ch)) + if !s.consumeIdentifier() { + return s.emit(ERROR) } if s.start == s.cursor { return s.scanUnknown() @@ -449,16 +448,15 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { } // Continue scanning identifier if no keyword match - for isIdentifier(ch) { - s.hasDigits = s.hasDigits || isDigit(ch) - ch = s.nextBy(utf8.RuneLen(ch)) + if !s.consumeIdentifier() { + return s.emit(ERROR) } if s.start == s.cursor { return s.scanUnknown() } - if ch == '(' { + if s.peek() == '(' { return s.emit(FUNCTION) } if s.config.DBMS == DBMSMySQL { @@ -467,19 +465,56 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { return s.emit(IDENT) } +func (s *Lexer) consumeIdentifier() bool { + for ch := s.peek(); isIdentifier(ch); ch = s.peek() { + if s.config.DBMS != DBMSMySQL && ch == '"' && s.cursor > s.start && s.src[s.cursor-1] == '.' { // opening quote right after a `.`, ex. schema."table-name" + if !s.consumeQuotedIdentifier('"') { + return false // returns false when no closing quote is found + } + continue + } + s.hasDigits = s.hasDigits || isDigit(ch) + s.nextBy(utf8.RuneLen(ch)) + } + return true +} + func (s *Lexer) scanDoubleQuotedIdentifier(delimiter rune) *Token { return s.scanDoubleQuotedIdentifierComponent(delimiter, true) } func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualified bool) *Token { + s.start = s.cursor + s.hasQuotes = true + s.isSimpleIdentifier = true + + if !s.consumeQuotedIdentifier(delimiter) { + s.hasQuotes = false + s.isSimpleIdentifier = false + return s.emit(ERROR) + } + + if joinQualified && s.config.DBMS == DBMSMySQL { + return s.checkForSpacesInIdentifier(QUOTED_IDENT) + } + + if joinQualified && s.peek() == '.' { // continue through an unquoted component, ex. "schema".table + s.isSimpleIdentifier = false + if !s.consumeIdentifier() { + s.hasQuotes = false + return s.emit(ERROR) + } + } + + return s.emit(QUOTED_IDENT) +} + +func (s *Lexer) consumeQuotedIdentifier(delimiter rune) bool { closingDelimiter := delimiter if delimiter == '[' { closingDelimiter = ']' } - s.start = s.cursor - s.hasQuotes = true - s.isSimpleIdentifier = true firstRune := true ch := s.next() // consume the opening quote specialCase := []rune{closingDelimiter, '.', delimiter} @@ -500,9 +535,7 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie break } if isEOF(ch) { - s.hasQuotes = false // if we hit EOF, we clear the quotes - s.isSimpleIdentifier = false - return s.emit(ERROR) + return false } s.hasDigits = s.hasDigits || isDigit(ch) if s.isSimpleIdentifier { @@ -521,10 +554,7 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie ch = s.nextBy(size) } s.next() // consume the closing quote (ASCII) - if joinQualified && s.config.DBMS == DBMSMySQL { - return s.checkForSpacesInIdentifier(QUOTED_IDENT) - } - return s.emit(QUOTED_IDENT) + return true } // scanMySQLQualifiedIdentifierComponent scans an unquoted identifier after a diff --git a/sqllexer_test.go b/sqllexer_test.go index 21cc684..2cbc7db 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1765,3 +1765,39 @@ func ExampleLexer() { fmt.Println(token) } } + +func TestLexerMixedQuoteQualifiedIdentifier(t *testing.T) { + t.Run("unquoted schema, quoted table", func(t *testing.T) { + lexer := New(`example_schema."orders-hyphen"`) + token := lexer.Scan() + if token.Type != IDENT || token.Value != `example_schema."orders-hyphen"` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } + }) + + t.Run("quoted schema, unquoted table", func(t *testing.T) { + lexer := New(`"example_schema".orders_underscore`) // unquoted hyphen would not be part of a table name, using underscore here + token := lexer.Scan() + if token.Type != QUOTED_IDENT || token.Value != `"example_schema".orders_underscore` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } + }) + + t.Run("quoted schema, quoted table", func(t *testing.T) { + lexer := New(`"example_schema"."orders-hyphen"`) + token := lexer.Scan() + if token.Type != QUOTED_IDENT || token.Value != `"example_schema"."orders-hyphen"` { + t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value) + } + if token := lexer.Scan(); token.Type != EOF { + t.Fatalf("expected EOF, got %v %q", token.Type, token.Value) + } + }) + +}