Fix: Handle mixed quote qualified identifiers

normalizer_test.go

@@ -1705,3 +1705,32 @@ func TestNormalizerDoesNotPinLargeBackingArrays(t *testing.T) {
 		return BackingArrayTestResult{SQL: sql, Metadata: metadata}, err
 	})
 }
+
+func TestNormalizeQualifiedUnquotedSchemaQuotedTable(t *testing.T) {
+	t.Run("unquoted schema, quoted table", func(t *testing.T) {
+		input := `SELECT * FROM example_schema."orders-dash"`
+		normalizer := NewNormalizer(WithCollectTables(true))
+		got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres))
+		assert.NoError(t, err)
+		assert.Equal(t, input, got)
+		assert.Equal(t, []string{`example_schema."orders-dash"`}, metadata.Tables)
+	})
+
+	t.Run("quoted schema, unquoted table", func(t *testing.T) {
+		input := `SELECT * FROM "example_schema".orders_underscore`
+		normalizer := NewNormalizer(WithCollectTables(true))
+		got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres))
+		assert.NoError(t, err)
+		assert.Equal(t, `SELECT * FROM example_schema.orders_underscore`, got)
+		assert.Equal(t, []string{`example_schema.orders_underscore`}, metadata.Tables)
+	})
+
+	t.Run("quoted schema, quoted table", func(t *testing.T) {
+		input := `SELECT * FROM "example_schema"."orders-hyphen"`
+		normalizer := NewNormalizer(WithCollectTables(true))
+		got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSPostgres))
+		assert.NoError(t, err)
+		assert.Equal(t, `SELECT * FROM example_schema.orders-hyphen`, got)
+		assert.Equal(t, []string{`example_schema.orders-hyphen`}, metadata.Tables)
+	})
+}

sqllexer.go

@@ -396,9 +396,8 @@ func (s *Lexer) scanIdentifier(ch rune) *Token {
 
 	// If first character is Unicode, skip trie lookup
 	if ch > 127 {
-		for isIdentifier(ch) {
-			s.hasDigits = s.hasDigits || isDigit(ch)
-			ch = s.nextBy(utf8.RuneLen(ch))
+		if !s.consumeIdentifier() {
+			return s.emit(ERROR)
 		}
 		if s.start == s.cursor {
 			return s.scanUnknown()
@@ -449,16 +448,15 @@ func (s *Lexer) scanIdentifier(ch rune) *Token {
 	}
 
 	// Continue scanning identifier if no keyword match
-	for isIdentifier(ch) {
-		s.hasDigits = s.hasDigits || isDigit(ch)
-		ch = s.nextBy(utf8.RuneLen(ch))
+	if !s.consumeIdentifier() {
+		return s.emit(ERROR)
 	}
 
 	if s.start == s.cursor {
 		return s.scanUnknown()
 	}
 
-	if ch == '(' {
+	if s.peek() == '(' {
 		return s.emit(FUNCTION)
 	}
 	if s.config.DBMS == DBMSMySQL {
@@ -467,19 +465,56 @@ func (s *Lexer) scanIdentifier(ch rune) *Token {
 	return s.emit(IDENT)
 }
 
+func (s *Lexer) consumeIdentifier() bool {
+	for ch := s.peek(); isIdentifier(ch); ch = s.peek() {
+		if s.config.DBMS != DBMSMySQL && ch == '"' && s.cursor > s.start && s.src[s.cursor-1] == '.' { // opening quote right after a `.`, ex. schema."table-name"
+			if !s.consumeQuotedIdentifier('"') {
+				return false // returns false when no closing quote is found
+			}
+			continue
+		}
+		s.hasDigits = s.hasDigits || isDigit(ch)
+		s.nextBy(utf8.RuneLen(ch))
+	}
+	return true
+}
+
 func (s *Lexer) scanDoubleQuotedIdentifier(delimiter rune) *Token {
 	return s.scanDoubleQuotedIdentifierComponent(delimiter, true)
 }
 
 func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualified bool) *Token {
+	s.start = s.cursor
+	s.hasQuotes = true
+	s.isSimpleIdentifier = true
+
+	if !s.consumeQuotedIdentifier(delimiter) {
+		s.hasQuotes = false
+		s.isSimpleIdentifier = false
+		return s.emit(ERROR)
+	}
+
+	if joinQualified && s.config.DBMS == DBMSMySQL {
+		return s.checkForSpacesInIdentifier(QUOTED_IDENT)
+	}
+
+	if joinQualified && s.peek() == '.' { // continue through an unquoted component, ex. "schema".table
+		s.isSimpleIdentifier = false
+		if !s.consumeIdentifier() {
+			s.hasQuotes = false
+			return s.emit(ERROR)
+		}
+	}
+
+	return s.emit(QUOTED_IDENT)
+}
+
+func (s *Lexer) consumeQuotedIdentifier(delimiter rune) bool {
 	closingDelimiter := delimiter
 	if delimiter == '[' {
 		closingDelimiter = ']'
 	}
 
-	s.start = s.cursor
-	s.hasQuotes = true
-	s.isSimpleIdentifier = true
 	firstRune := true
 	ch := s.next() // consume the opening quote
 	specialCase := []rune{closingDelimiter, '.', delimiter}
@@ -500,9 +535,7 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie
 			break
 		}
 		if isEOF(ch) {
-			s.hasQuotes = false // if we hit EOF, we clear the quotes
-			s.isSimpleIdentifier = false
-			return s.emit(ERROR)
+			return false
 		}
 		s.hasDigits = s.hasDigits || isDigit(ch)
 		if s.isSimpleIdentifier {
@@ -521,10 +554,7 @@ func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualifie
 		ch = s.nextBy(size)
 	}
 	s.next() // consume the closing quote (ASCII)
-	if joinQualified && s.config.DBMS == DBMSMySQL {
-		return s.checkForSpacesInIdentifier(QUOTED_IDENT)
-	}
-	return s.emit(QUOTED_IDENT)
+	return true
 }
 
 // scanMySQLQualifiedIdentifierComponent scans an unquoted identifier after a

sqllexer_test.go

@@ -1765,3 +1765,39 @@ func ExampleLexer() {
 		fmt.Println(token)
 	}
 }
+
+func TestLexerMixedQuoteQualifiedIdentifier(t *testing.T) {
+	t.Run("unquoted schema, quoted table", func(t *testing.T) {
+		lexer := New(`example_schema."orders-hyphen"`)
+		token := lexer.Scan()
+		if token.Type != IDENT || token.Value != `example_schema."orders-hyphen"` {
+			t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value)
+		}
+		if token := lexer.Scan(); token.Type != EOF {
+			t.Fatalf("expected EOF, got %v %q", token.Type, token.Value)
+		}
+	})
+
+	t.Run("quoted schema, unquoted table", func(t *testing.T) {
+		lexer := New(`"example_schema".orders_underscore`) // unquoted hyphen would not be part of a table name, using underscore here
+		token := lexer.Scan()
+		if token.Type != QUOTED_IDENT || token.Value != `"example_schema".orders_underscore` {
+			t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value)
+		}
+		if token := lexer.Scan(); token.Type != EOF {
+			t.Fatalf("expected EOF, got %v %q", token.Type, token.Value)
+		}
+	})
+
+	t.Run("quoted schema, quoted table", func(t *testing.T) {
+		lexer := New(`"example_schema"."orders-hyphen"`)
+		token := lexer.Scan()
+		if token.Type != QUOTED_IDENT || token.Value != `"example_schema"."orders-hyphen"` {
+			t.Fatalf("expected complete qualified identifier, got %v %q", token.Type, token.Value)
+		}
+		if token := lexer.Scan(); token.Type != EOF {
+			t.Fatalf("expected EOF, got %v %q", token.Type, token.Value)
+		}
+	})
+
+}