diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd708786..5df257c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Every saved group lost when one unreadable entry stopped the whole list decoding. (#1311) - Two Macs re-uploading the whole group list to each other after a single group changed. (#1311) - Deleting a group that a broken sync left in a loop also deleting the group it pointed at. (#1311) +- "Operator does not exist" from a Contains, Starts with, Ends with or Regex filter on a PostgreSQL uuid, enum, number, date or json column. +- "Function lower does not exist" from an ignore-case filter on a PostgreSQL column that is not text. +- Is empty filter on a PostgreSQL array column. +- MongoDB collection named like a `db` method, such as `stats` or `version`, failing to open, save or export. +- Row count missing after a MongoDB raw filter written in shell syntax. ## [0.71.0] - 2026-09-02 diff --git a/Plugins/MQLExportPlugin/MQLExportHelpers.swift b/Plugins/MQLExportPlugin/MQLExportHelpers.swift index d0a5936d5..857f95a8b 100644 --- a/Plugins/MQLExportPlugin/MQLExportHelpers.swift +++ b/Plugins/MQLExportPlugin/MQLExportHelpers.swift @@ -8,21 +8,8 @@ import TableProNumberFormatting import TableProPluginKit enum MQLExportHelpers { - static func escapeJSIdentifier(_ name: String) -> String { - guard let firstChar = name.first, - !firstChar.isNumber, - name.allSatisfy({ $0.isLetter || $0.isNumber || $0 == "_" }) else { - return "[\"\(PluginExportUtilities.escapeJSONString(name))\"]" - } - return name - } - static func collectionAccessor(for name: String) -> String { - let escaped = escapeJSIdentifier(name) - if escaped.hasPrefix("[") { - return "db\(escaped)" - } - return "db.\(escaped)" + MongoCollectionAccessor.expression(for: name) } static func mqlBinaryValue(for data: Data, subtype: UInt8) -> String { diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 211e691a6..064fd4cea 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -14,6 +14,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var scriptRuntime: MongoScriptRuntime? private var currentDb: String private let columnKindLock = NSLock() + private let rawFilterNormalizer = MongoDBRawFilterNormalizer() private var columnKindsByCollection: [String: [String: BsonValueKind]] = [:] private var fieldPathKindsByCollection: [String: [String: BsonValueKind]] = [:] @@ -432,7 +433,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { throw MongoDBPluginError.notConnected } - let filterJson = MongoDBQueryBuilder(columnKinds: filterKinds(for: table)) + let filterJson = filterQueryBuilder(for: table) .buildFilterDocument(from: filters, logicMode: logicMode) let count = try await conn.countDocuments( database: currentDb, collection: table, filter: filterJson, background: background @@ -494,9 +495,8 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { opts.append("\"name\": \"\(name)\"") let optsJson = "{\(opts.joined(separator: ", "))}" - let escapedTable = table.replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\"", with: "\\\"") - sections.append("db[\"\(escapedTable)\"].createIndex(\(keyJson), \(optsJson))") + let accessor = MongoCollectionAccessor.expression(for: table) + sections.append("\(accessor).createIndex(\(keyJson), \(optsJson))") } } } catch { @@ -699,7 +699,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { offset: Int, columnKinds: [String: PluginColumnKind] ) -> String? { - let builder = MongoDBQueryBuilder(columnKinds: filterKinds(for: table)) + let builder = filterQueryBuilder(for: table) return builder.buildFilteredQuery( collection: table, queryFilters: queryFilters, logicMode: logicMode, sortColumns: sortColumns, columns: columns, limit: limit, offset: offset @@ -923,6 +923,14 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + private func filterQueryBuilder(for collection: String) -> MongoDBQueryBuilder { + let normalizer = rawFilterNormalizer + return MongoDBQueryBuilder( + columnKinds: filterKinds(for: collection), + rawFilterNormalizer: { normalizer.normalize($0) } + ) + } + /// Two databases can hold a collection of the same name with different field types. private func columnKindKey(_ collection: String) -> String { "\(currentDb)\u{0}\(collection)" diff --git a/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index 86484cca9..753b3f925 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -21,8 +21,16 @@ struct MongoDBQueryBuilder { let columnKinds: [String: BsonValueKind] - init(columnKinds: [String: BsonValueKind] = [:]) { + /// Rewrites a raw filter row into canonical Extended JSON, so `find` and `countDocuments` + /// receive the same document. Without one the row's text is used as typed. + let rawFilterNormalizer: (@Sendable (String) -> String?)? + + init( + columnKinds: [String: BsonValueKind] = [:], + rawFilterNormalizer: (@Sendable (String) -> String?)? = nil + ) { self.columnKinds = columnKinds + self.rawFilterNormalizer = rawFilterNormalizer } // MARK: - Base Query @@ -155,7 +163,8 @@ struct MongoDBQueryBuilder { guard filter.column == Self.rawFilterColumn else { return nil } let trimmed = filter.value.trimmingCharacters(in: .whitespacesAndNewlines) guard trimmed.hasPrefix("{"), trimmed.hasSuffix("}") else { return nil } - return MongoDBFilterClause(key: "$and", body: "[\(trimmed)]") + let document = rawFilterNormalizer?(trimmed) ?? trimmed + return MongoDBFilterClause(key: "$and", body: "[\(document)]") } /// One `$elemMatch` per array prefix. Every condition is re-keyed to its path relative to the @@ -194,12 +203,7 @@ struct MongoDBQueryBuilder { } private static func mongoCollectionAccessor(_ name: String) -> String { - guard let firstChar = name.first, - !firstChar.isNumber, - name.allSatisfy({ $0.isLetter || $0.isNumber || $0 == "_" }) else { - return "db[\"\(escapeJsonString(name))\"]" - } - return "db.\(name)" + MongoCollectionAccessor.expression(for: name) } private func buildClause(for filter: PluginQueryFilter, field rawField: String) -> MongoDBFilterClause? { diff --git a/Plugins/MongoDBDriverPlugin/MongoDBRawFilterNormalizer.swift b/Plugins/MongoDBDriverPlugin/MongoDBRawFilterNormalizer.swift new file mode 100644 index 000000000..ef005804c --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBRawFilterNormalizer.swift @@ -0,0 +1,97 @@ +// +// MongoDBRawFilterNormalizer.swift +// MongoDBDriverPlugin +// + +import Foundation +import JavaScriptCore + +/// Turns a filter document the user typed in shell syntax into canonical Extended JSON. +/// +/// A raw filter row reaches two consumers with two parsers: `find` evaluates it as JavaScript, +/// where `{status: "active", _id: ObjectId("…")}` is fine, and `countDocuments` hands the same +/// text to libmongoc's JSON parser, where it is not. Serializing through the shell's own `EJSON` +/// once, up front, gives both the document the user meant. +/// +/// The prelude is the same one the shell runs, with the host stubbed: it answers the one call +/// the prelude makes while loading and refuses every other, so anything that would need the +/// server, such as `ObjectId()` with no argument, comes back as `nil` and the caller keeps the +/// text it had. +/// +/// The text is JavaScript, so it can also loop forever, and JavaScriptCore's public API cannot +/// interrupt it. Like `MongoScriptRuntime`, each engine runs on a queue of its own: a document +/// that misses the deadline is answered `nil`, its engine is abandoned with the queue it wedged, +/// and the next call builds a fresh one. +final class MongoDBRawFilterNormalizer: @unchecked Sendable { + private static let deadline: TimeInterval = 2 + + private final class Engine: @unchecked Sendable { + let queue: DispatchQueue + let context: JSContext + + init(queue: DispatchQueue, context: JSContext) { + self.queue = queue + self.context = context + } + } + + private final class Outcome: @unchecked Sendable { + var value: String? + } + + private let lock = NSLock() + private var engine: Engine? + private var generation = 0 + + func normalize(_ document: String) -> String? { + lock.lock() + defer { lock.unlock() } + guard let engine = preparedEngine() else { return nil } + + let outcome = Outcome() + let finished = DispatchSemaphore(value: 0) + engine.queue.async { + outcome.value = Self.serialize(document, in: engine.context) + finished.signal() + } + guard finished.wait(timeout: .now() + Self.deadline) == .success else { + self.engine = nil + return nil + } + return outcome.value + } + + private static func serialize(_ document: String, in context: JSContext) -> String? { + context.exception = nil + let value = context.evaluateScript("__ejson((\(document)))") + guard context.exception == nil, let value, value.isString else { return nil } + return value.toString() + } + + private func preparedEngine() -> Engine? { + if let engine { return engine } + guard let context = JSContext(virtualMachine: JSVirtualMachine()) else { return nil } + let host: @convention(block) (String) -> String = { request in Self.answer(request) } + let swallowOutput: @convention(block) (String) -> Bool = { _ in true } + context.setObject(host, forKeyedSubscript: "__tp_exec" as NSString) + context.setObject(swallowOutput, forKeyedSubscript: "__tp_print" as NSString) + context.evaluateScript(MongoScriptPrelude.source) + guard context.exception == nil else { return nil } + + generation += 1 + let queue = DispatchQueue( + label: "com.TablePro.mongodb.rawfilter.\(generation)", qos: .userInitiated + ) + let built = Engine(queue: queue, context: context) + engine = built + return built + } + + private static func answer(_ request: String) -> String { + let parsed = try? JSONSerialization.jsonObject(with: Data(request.utf8)) as? [String: Any] + guard parsed?["op"] as? String == "currentDatabase" else { + return MongoScriptJson.failure(message: "The server is not reachable from a filter", code: 0) + } + return MongoScriptJson.success(MongoScriptJson.jsonString("")) + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index 3cd8459b6..6c0a4bb32 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -18,9 +18,8 @@ struct MongoDBStatementGenerator { let columns: [String] var columnKinds: [String: BsonValueKind] = [:] - /// Collection accessor using bracket notation for safety with dotted names private var collectionAccessor: String { - "db[\"\(escapeJsonString(collectionName))\"]" + MongoCollectionAccessor.expression(for: collectionName) } /// Index of "_id" field in the columns array (used as primary key equivalent) diff --git a/Plugins/PostgreSQLDriverPlugin/PostgreSQLDialect.swift b/Plugins/PostgreSQLDriverPlugin/PostgreSQLDialect.swift index 183c6a8e0..eab3ef6b8 100644 --- a/Plugins/PostgreSQLDriverPlugin/PostgreSQLDialect.swift +++ b/Plugins/PostgreSQLDriverPlugin/PostgreSQLDialect.swift @@ -13,7 +13,8 @@ enum PostgreSQLDialect { likeEscapeStyle: .explicit, paginationStyle: .limit, caseSensitivityStyle: .ilikeOperator, - operators: operators + operators: operators, + textCastTypeName: "TEXT" ) static let keywords: Set = reservedKeywords diff --git a/Plugins/TableProPluginKit/MongoCollectionAccessor.swift b/Plugins/TableProPluginKit/MongoCollectionAccessor.swift new file mode 100644 index 000000000..45d6cad62 --- /dev/null +++ b/Plugins/TableProPluginKit/MongoCollectionAccessor.swift @@ -0,0 +1,55 @@ +// +// MongoCollectionAccessor.swift +// TableProPluginKit +// + +import Foundation + +/// Spells the shell expression that reaches a collection by name. +/// +/// `db.` and `db[""]` both go through the `db` object's property lookup, and in +/// mongosh as in TablePro's own shell that lookup answers a method before a collection. So a +/// collection called `stats` or `version` comes back as a function, and `.find()` on it is a +/// TypeError. `db.getCollection("")` is the one spelling that cannot be shadowed. +public enum MongoCollectionAccessor { + public static func expression(for name: String) -> String { + guard isPlainIdentifier(name), !isShadowedByDatabaseMember(name) else { + return "db.getCollection(\"\(PluginExportUtilities.escapeJSONString(name))\")" + } + return "db.\(name)" + } + + public static func unescape(_ escaped: String) -> String { + let quoted = Data("\"\(escaped)\"".utf8) + return (try? JSONDecoder().decode(String.self, from: quoted)) ?? escaped + } + + public static func isShadowedByDatabaseMember(_ name: String) -> Bool { + name.hasPrefix("__") || databaseMemberNames.contains(name) + } + + private static func isPlainIdentifier(_ name: String) -> Bool { + guard let first = name.first, !first.isNumber else { return false } + return name.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" } + } + + /// Every method mongosh puts on `db`, plus what `Object.prototype` gives any JavaScript value. + public static let databaseMemberNames: Set = [ + "adminCommand", "aggregate", "auth", "changeUserPassword", "checkMetadataConsistency", + "commandHelp", "createCollection", "createRole", "createUser", "createView", "currentOp", + "disableFreeMonitoring", "dropAllRoles", "dropAllUsers", "dropDatabase", "dropRole", + "dropUser", "enableFreeMonitoring", "fsyncLock", "fsyncUnlock", "getCollection", + "getCollectionInfos", "getCollectionNames", "getFreeMonitoringStatus", "getLastError", + "getLastErrorObj", "getLogComponents", "getMongo", "getName", "getProfilingLevel", + "getProfilingStatus", "getReplicationInfo", "getRole", "getRoles", "getSiblingDB", + "getUser", "getUsers", "grantPrivilegesToRole", "grantRolesToRole", "grantRolesToUser", + "hello", "help", "hostInfo", "isMaster", "killOp", "listCommands", "logout", + "printCollectionStats", "printReplicationInfo", "printSecondaryReplicationInfo", + "printShardingStatus", "printSlaveReplicationInfo", "removeUser", "revokePrivilegesFromRole", + "revokeRolesFromRole", "revokeRolesFromUser", "rotateCertificates", "runCommand", + "serverBuildInfo", "serverCmdLineOpts", "serverStatus", "setLogLevel", "setProfilingLevel", + "shutdownServer", "sql", "stats", "updateRole", "updateUser", "version", "watch", + "constructor", "hasOwnProperty", "isPrototypeOf", "propertyIsEnumerable", + "toLocaleString", "toString", "valueOf" + ] +} diff --git a/Plugins/TableProPluginKit/SQLDialectDescriptor.swift b/Plugins/TableProPluginKit/SQLDialectDescriptor.swift index 8e95c712e..8666824df 100644 --- a/Plugins/TableProPluginKit/SQLDialectDescriptor.swift +++ b/Plugins/TableProPluginKit/SQLDialectDescriptor.swift @@ -84,6 +84,12 @@ public struct SQLDialectDescriptor: Sendable { public let caseSensitivityStyle: CaseSensitivityStyle public let caseFoldFunction: String + // Pattern matching on a non-character column + /// The type a column that is not character data is cast to before `LIKE`, a regex or a case + /// fold. `nil` means the engine coerces the operand itself. PostgreSQL does not: `uuid ~~ unknown` + /// and `lower(integer)` are both "operator does not exist". + public let textCastTypeName: String? + // Authoring public let operators: [SQLOperatorDescriptor] @@ -191,6 +197,7 @@ public struct SQLDialectDescriptor: Sendable { ) } + @_disfavoredOverload public init( identifierQuote: String, keywords: Set, @@ -207,6 +214,44 @@ public struct SQLDialectDescriptor: Sendable { caseSensitivityStyle: CaseSensitivityStyle = .unsupported, caseFoldFunction: String = SQLDialectDescriptor.defaultCaseFoldFunction, operators: [SQLOperatorDescriptor] = [] + ) { + self.init( + identifierQuote: identifierQuote, + keywords: keywords, + functions: functions, + dataTypes: dataTypes, + tableOptions: tableOptions, + regexSyntax: regexSyntax, + booleanLiteralStyle: booleanLiteralStyle, + likeEscapeStyle: likeEscapeStyle, + paginationStyle: paginationStyle, + offsetFetchOrderBy: offsetFetchOrderBy, + requiresBackslashEscaping: requiresBackslashEscaping, + autoLimitStyle: autoLimitStyle, + caseSensitivityStyle: caseSensitivityStyle, + caseFoldFunction: caseFoldFunction, + operators: operators, + textCastTypeName: nil + ) + } + + public init( + identifierQuote: String, + keywords: Set, + functions: Set, + dataTypes: Set, + tableOptions: [String] = [], + regexSyntax: RegexSyntax = .unsupported, + booleanLiteralStyle: BooleanLiteralStyle = .numeric, + likeEscapeStyle: LikeEscapeStyle = .explicit, + paginationStyle: PaginationStyle = .limit, + offsetFetchOrderBy: String = "ORDER BY (SELECT NULL)", + requiresBackslashEscaping: Bool = false, + autoLimitStyle: AutoLimitStyle = .limit, + caseSensitivityStyle: CaseSensitivityStyle = .unsupported, + caseFoldFunction: String = SQLDialectDescriptor.defaultCaseFoldFunction, + operators: [SQLOperatorDescriptor] = [], + textCastTypeName: String? ) { self.identifierQuote = identifierQuote self.keywords = keywords @@ -223,6 +268,7 @@ public struct SQLDialectDescriptor: Sendable { self.caseSensitivityStyle = caseSensitivityStyle self.caseFoldFunction = caseFoldFunction self.operators = operators + self.textCastTypeName = textCastTypeName } public static let defaultCaseFoldFunction = "LOWER" @@ -246,7 +292,8 @@ public struct SQLDialectDescriptor: Sendable { autoLimitStyle: autoLimitStyle, caseSensitivityStyle: style, caseFoldFunction: caseFoldFunction, - operators: operators + operators: operators, + textCastTypeName: textCastTypeName ) } } diff --git a/TablePro/Core/Database/FilterSQLGenerator.swift b/TablePro/Core/Database/FilterSQLGenerator.swift index c3ad68f52..1def94bbd 100644 --- a/TablePro/Core/Database/FilterSQLGenerator.swift +++ b/TablePro/Core/Database/FilterSQLGenerator.swift @@ -75,8 +75,8 @@ struct FilterSQLGenerator { return "\(quotedColumn) IS NULL" case .value(let literal): return generateComparisonCondition( - column: quotedColumn, literal: literal, rawValue: filter.value, - negated: false, folding: folding + column: quotedColumn, columnType: columnType, literal: literal, + rawValue: filter.value, negated: false, folding: folding ) } @@ -86,14 +86,14 @@ struct FilterSQLGenerator { return "\(quotedColumn) IS NOT NULL" case .value(let literal): return generateComparisonCondition( - column: quotedColumn, literal: literal, rawValue: filter.value, - negated: true, folding: folding + column: quotedColumn, columnType: columnType, literal: literal, + rawValue: filter.value, negated: true, folding: folding ) } case .contains, .notContains, .startsWith, .endsWith: return generateLikeFamilyCondition( - column: quotedColumn, filter: filter, folding: folding + column: patternOperand(quotedColumn, columnType: columnType), filter: filter, folding: folding ) case .greaterThan: @@ -118,13 +118,13 @@ struct FilterSQLGenerator { guard ColumnTypeSQLQuoting.supportsEmptyStringComparison(columnType) else { return "\(quotedColumn) IS NULL" } - return "(\(quotedColumn) IS NULL OR \(quotedColumn) = '')" + return "(\(quotedColumn) IS NULL OR \(patternOperand(quotedColumn, columnType: columnType)) = '')" case .isNotEmpty: guard ColumnTypeSQLQuoting.supportsEmptyStringComparison(columnType) else { return "\(quotedColumn) IS NOT NULL" } - return "(\(quotedColumn) IS NOT NULL AND \(quotedColumn) != '')" + return "(\(quotedColumn) IS NOT NULL AND \(patternOperand(quotedColumn, columnType: columnType)) != '')" case .inList: return generateInCondition( @@ -145,17 +145,29 @@ struct FilterSQLGenerator { return "\(quotedColumn) BETWEEN \(lower) AND \(upper)" case .regex: + let operand = patternOperand(quotedColumn, columnType: columnType) guard dialect.regexSyntax != .unsupported else { let pattern = "'%\(escapeSQLQuote(filter.value))%'" - return "\(folding.foldingLikeOperand(quotedColumn)) \(folding.likeKeyword) " + return "\(folding.foldingLikeOperand(operand)) \(folding.likeKeyword) " + folding.foldingLikeOperand(pattern) } return generateRegexCondition( - column: quotedColumn, pattern: filter.value, ignoresCase: !filter.isCaseSensitive + column: operand, pattern: filter.value, ignoresCase: !filter.isCaseSensitive ) } } + /// The operand `LIKE`, a regex and a case fold see. An engine that names a + /// `textCastTypeName` refuses those on anything but character data, so every other column + /// is cast first, including one whose type the grid has not resolved yet, because a cast + /// to text is valid on every column and a bare operand is not. A plain `=` keeps the + /// column's own type and its index. + private func patternOperand(_ column: String, columnType: ColumnType?) -> String { + guard let castType = dialect.textCastTypeName, + !ColumnTypeSQLQuoting.isCharacterType(columnType) else { return column } + return "CAST(\(column) AS \(castType))" + } + // MARK: - Case Sensitivity private func caseFolding(for filter: TableFilter, columnType: ColumnType?) -> PluginSQLCaseFolding { @@ -170,9 +182,12 @@ struct FilterSQLGenerator { } /// Folding a non-text column is a type error on strict engines, so only fold - /// columns that are text or whose type the grid has not resolved. + /// columns that are text or whose type the grid has not resolved, unless the + /// dialect casts the operand to text first, which makes every column foldable. private func allowsCaseFolding(_ columnType: ColumnType?) -> Bool { - columnType == nil || ColumnTypeSQLQuoting.isKnownTextLike(columnType) + columnType == nil + || ColumnTypeSQLQuoting.isKnownTextLike(columnType) + || dialect.textCastTypeName != nil } private func foldedComparison(_ literal: String, folding: PluginSQLCaseFolding) -> String? { @@ -195,18 +210,20 @@ struct FilterSQLGenerator { let parsed = parseListValues(values) guard !parsed.isEmpty else { return nil } - var nonNullValues: [String] = [] + var literals: [String] = [] var hasNull = false for item in parsed { switch renderLiteral(item, columnType: columnType) { case .null: hasNull = true case .value(let literal): - nonNullValues.append(folding.foldingComparison(literal)) + literals.append(literal) } } - let foldedColumn = folding.foldingComparison(column) + let foldsList = folding.foldsComparisonOperands && literals.allSatisfy { $0.hasPrefix("'") } + let nonNullValues = foldsList ? literals.map(folding.fold) : literals + let foldedColumn = foldsList ? folding.fold(patternOperand(column, columnType: columnType)) : column let inClause: String? = nonNullValues.isEmpty ? nil : { let list = nonNullValues.joined(separator: ", ") return negated @@ -283,6 +300,7 @@ struct FilterSQLGenerator { private func generateComparisonCondition( column: String, + columnType: ColumnType?, literal: String, rawValue: String, negated: Bool, @@ -292,14 +310,18 @@ struct FilterSQLGenerator { let pattern = PluginSQLRegexPattern.pattern( matchingLiteral: rawValue, anchoring: .exact, ignoresCase: false ) - let condition = generateRegexCondition(column: column, pattern: pattern, ignoresCase: true) + let condition = generateRegexCondition( + column: patternOperand(column, columnType: columnType), pattern: pattern, ignoresCase: true + ) return negated ? "NOT (\(condition))" : condition } let operatorText = negated ? "!=" : "=" guard let foldedValue = foldedComparison(literal, folding: folding) else { return "\(column) \(operatorText) \(literal)" } - let foldedColumn = folding.foldsComparisonOperands ? folding.fold(column) : column + let foldedColumn = folding.foldsComparisonOperands + ? folding.fold(patternOperand(column, columnType: columnType)) + : column return "\(foldedColumn) \(operatorText) \(foldedValue)" } diff --git a/TablePro/Core/Services/Query/QuerySqlParser.swift b/TablePro/Core/Services/Query/QuerySqlParser.swift index abdefdc43..c7dad4232 100644 --- a/TablePro/Core/Services/Query/QuerySqlParser.swift +++ b/TablePro/Core/Services/Query/QuerySqlParser.swift @@ -12,6 +12,11 @@ enum QuerySqlParser { options: [] ) + private static let mongoGetCollectionRegex = try? NSRegularExpression( + pattern: #"^\s*db\.getCollection\(\s*"((?:[^"\\]|\\.)*)"\s*\)"#, + options: [] + ) + /// The table a result grid may be edited through, or `nil` when the statement reads from /// anything other than exactly one table. /// @@ -33,6 +38,12 @@ enum QuerySqlParser { let nsRange = NSRange(sql.startIndex..., in: sql) + if let regex = mongoGetCollectionRegex, + let match = regex.firstMatch(in: sql, options: [], range: nsRange), + let range = Range(match.range(at: 1), in: sql) { + return MongoCollectionAccessor.unescape(String(sql[range])) + } + if let regex = mongoBracketCollectionRegex, let match = regex.firstMatch(in: sql, options: [], range: nsRange), let range = Range(match.range(at: 1), in: sql) { diff --git a/TablePro/Core/Utilities/SQL/ColumnTypeSQLQuoting.swift b/TablePro/Core/Utilities/SQL/ColumnTypeSQLQuoting.swift index feb19965b..3a8d9bcce 100644 --- a/TablePro/Core/Utilities/SQL/ColumnTypeSQLQuoting.swift +++ b/TablePro/Core/Utilities/SQL/ColumnTypeSQLQuoting.swift @@ -28,13 +28,30 @@ internal enum ColumnTypeSQLQuoting { static func isKnownTextLike(_ type: ColumnType?) -> Bool { guard let type else { return false } switch type { - case .text, .enumType, .set, .array: + case .text, .enumType, .set: return true - case .integer, .decimal, .date, .timestamp, .datetime, .boolean, .blob, .json, .spatial: + case .integer, .decimal, .date, .timestamp, .datetime, .boolean, .blob, .json, .spatial, .array: return false } } + /// Whether the column holds character data an engine compares with `LIKE` natively. A `.text` + /// column is only what the classifier could not place elsewhere, so `uuid`, `inet` and every + /// unknown type land there too, and their raw name is what separates them from `varchar`. + static func isCharacterType(_ type: ColumnType?) -> Bool { + guard case let .text(rawType)? = type else { return false } + guard let rawType else { return true } + let base = rawType.prefix { $0 != "(" } + .trimmingCharacters(in: .whitespaces) + .uppercased() + if characterBaseNames.contains(base) { return true } + return base.contains("CHAR") || base.hasSuffix("TEXT") + } + + private static let characterBaseNames: Set = [ + "STRING", "FIXEDSTRING", "CLOB", "NCLOB", "NAME", "CITEXT" + ] + static func supportsEmptyStringComparison(_ type: ColumnType?) -> Bool { guard let type else { return true } return isKnownTextLike(type) diff --git a/TableProTests/Core/Database/FilterSQLGeneratorColumnTypeTests.swift b/TableProTests/Core/Database/FilterSQLGeneratorColumnTypeTests.swift index 87ab85276..4edf461c1 100644 --- a/TableProTests/Core/Database/FilterSQLGeneratorColumnTypeTests.swift +++ b/TableProTests/Core/Database/FilterSQLGeneratorColumnTypeTests.swift @@ -236,4 +236,136 @@ struct FilterSQLGeneratorColumnTypeTests { #expect(contains == "`id` LIKE '%68%'") #expect(startsWith == "`code` LIKE '68%'") } + + // MARK: - Text cast for pattern matching + + private static let castingPostgres = SQLDialectDescriptor( + identifierQuote: "\"", keywords: [], functions: [], dataTypes: [], + regexSyntax: .tilde, booleanLiteralStyle: .truefalse, + likeEscapeStyle: .explicit, paginationStyle: .limit, + caseSensitivityStyle: .ilikeOperator, textCastTypeName: "TEXT" + ) + + private static func castingCondition( + column: String, + type: ColumnType?, + op: FilterOperator, + value: String, + isCaseSensitive: Bool? = nil + ) -> String? { + let generator = FilterSQLGenerator( + dialect: castingPostgres, + columns: type == nil ? [] : [column], + columnTypes: type.map { [$0] } ?? [] + ) + let filter = TableFilter( + columnName: column, filterOperator: op, value: value, isCaseSensitive: isCaseSensitive + ) + return generator.generateCondition(from: filter) + } + + @Test("Contains on a uuid column is cast to text before ILIKE") + func uuidContainsIsCastToText() { + let result = Self.castingCondition( + column: "id", type: .text(rawType: "uuid"), op: .contains, value: "ab" + ) + #expect(result == "CAST(\"id\" AS TEXT) ILIKE '%ab%' ESCAPE '!'") + } + + @Test("Contains on a character column is not cast") + func varcharContainsKeepsTheColumn() { + for rawType in ["varchar", "character varying(255)", "text", "bpchar", "citext", "name"] { + let result = Self.castingCondition( + column: "name", type: .text(rawType: rawType), op: .contains, value: "ab" + ) + #expect(result == "\"name\" ILIKE '%ab%' ESCAPE '!'", "\(rawType)") + } + } + + @Test("Starts with on an integer column is cast to text") + func integerStartsWithIsCastToText() { + let result = Self.castingCondition( + column: "id", type: .integer(rawType: "integer"), op: .startsWith, value: "68" + ) + #expect(result == "CAST(\"id\" AS TEXT) ILIKE '68%' ESCAPE '!'") + } + + @Test("Ignoring case on an enum folds the cast, matching case keeps the enum comparison") + func enumEqualsFoldsOnlyTheCast() { + let folded = Self.castingCondition( + column: "status", type: .enumType(rawType: "ENUM(mood)", values: nil), + op: .equal, value: "happy", isCaseSensitive: false + ) + let exact = Self.castingCondition( + column: "status", type: .enumType(rawType: "ENUM(mood)", values: nil), + op: .equal, value: "happy" + ) + #expect(folded == "LOWER(CAST(\"status\" AS TEXT)) = LOWER('happy')") + #expect(exact == "\"status\" = 'happy'") + } + + @Test("Regex on a timestamp column is cast to text") + func timestampRegexIsCastToText() { + let result = Self.castingCondition( + column: "created_at", type: .timestamp(rawType: "timestamptz"), op: .regex, value: "^2024" + ) + #expect(result == "CAST(\"created_at\" AS TEXT) ~ '^2024'") + } + + @Test("Is empty on a uuid column compares the cast, on an array only null") + func isEmptyCastsOrFallsBackToNull() { + let uuid = Self.castingCondition( + column: "id", type: .text(rawType: "uuid"), op: .isEmpty, value: "" + ) + let array = Self.castingCondition( + column: "tags", type: .array(rawType: "text[]", element: .text(rawType: "text")), + op: .isEmpty, value: "" + ) + #expect(uuid == "(\"id\" IS NULL OR CAST(\"id\" AS TEXT) = '')") + #expect(array == "\"tags\" IS NULL") + } + + @Test("Contains on an array column searches its text form") + func arrayContainsIsCastToText() { + let result = Self.castingCondition( + column: "tags", type: .array(rawType: "text[]", element: .text(rawType: "text")), + op: .contains, value: "red" + ) + #expect(result == "CAST(\"tags\" AS TEXT) ILIKE '%red%' ESCAPE '!'") + } + + @Test("In list ignoring case on a uuid column folds the cast") + func uuidInListIgnoringCaseFoldsTheCast() { + let result = Self.castingCondition( + column: "id", type: .text(rawType: "uuid"), op: .inList, value: "a, b", isCaseSensitive: false + ) + #expect(result == "LOWER(CAST(\"id\" AS TEXT)) IN (LOWER('a'), LOWER('b'))") + } + + @Test("A column of unknown type is cast, because a cast is valid on every column") + func unknownColumnTypeIsCast() { + let result = Self.castingCondition(column: "id", type: nil, op: .contains, value: "ab") + #expect(result == "CAST(\"id\" AS TEXT) ILIKE '%ab%' ESCAPE '!'") + } + + @Test("In list ignoring case on an integer column keeps the numbers unfolded") + func integerInListIgnoringCaseIsNotFolded() { + let result = Self.castingCondition( + column: "id", type: .integer(rawType: "integer"), op: .inList, value: "1, 2", isCaseSensitive: false + ) + let mixed = Self.castingCondition( + column: "id", type: .integer(rawType: "integer"), op: .inList, value: "1, x", isCaseSensitive: false + ) + #expect(result == "\"id\" IN (1, 2)") + #expect(mixed == "\"id\" IN (1, 'x')") + } + + @Test("A dialect without a text cast type leaves a non-text column alone") + func dialectWithoutCastTypeDoesNotCast() { + let result = Self.condition( + column: "id", type: .text(rawType: "uuid"), op: .contains, value: "ab", + dialect: Self.postgresqlDialect + ) + #expect(result == "\"id\" LIKE '%ab%' ESCAPE '!'") + } } diff --git a/TableProTests/Core/Plugins/SQLDialectDescriptorTests.swift b/TableProTests/Core/Plugins/SQLDialectDescriptorTests.swift index 1846c0b0f..6fb7a786a 100644 --- a/TableProTests/Core/Plugins/SQLDialectDescriptorTests.swift +++ b/TableProTests/Core/Plugins/SQLDialectDescriptorTests.swift @@ -80,4 +80,22 @@ final class SQLDialectDescriptorTests: XCTestCase { XCTAssertTrue(adapter.isDataType("int")) XCTAssertFalse(adapter.isDataType("NONEXISTENT")) } + + func testTextCastTypeNameDefaultsToNil() { + let descriptor = SQLDialectDescriptor( + identifierQuote: "`", keywords: [], functions: [], dataTypes: [] + ) + XCTAssertNil(descriptor.textCastTypeName) + } + + func testTextCastTypeNameSurvivesCaseSensitivityRestyle() { + let descriptor = SQLDialectDescriptor( + identifierQuote: "\"", keywords: [], functions: [], dataTypes: [], + caseSensitivityStyle: .ilikeOperator, textCastTypeName: "TEXT" + ) + let restyled = descriptor.withCaseSensitivityStyle(.caseFoldFunction) + XCTAssertEqual(descriptor.textCastTypeName, "TEXT") + XCTAssertEqual(restyled.textCastTypeName, "TEXT") + XCTAssertEqual(restyled.caseSensitivityStyle, .caseFoldFunction) + } } diff --git a/TableProTests/Core/Services/Query/QueryExecutorTests.swift b/TableProTests/Core/Services/Query/QueryExecutorTests.swift index 7e6f22ca2..391902185 100644 --- a/TableProTests/Core/Services/Query/QueryExecutorTests.swift +++ b/TableProTests/Core/Services/Query/QueryExecutorTests.swift @@ -50,6 +50,16 @@ struct QueryExecutorTests { #expect(name == "user logs") } + @Test("extractTableName parses MQL getCollection notation") + func extractTableNameMQLGetCollection() { + let plain = QuerySqlParser.extractTableName(from: #"db.getCollection("user logs").find({})"#) + let escaped = QuerySqlParser.extractTableName(from: #"db.getCollection("say\"hi").find({})"#) + let shadowed = QuerySqlParser.extractTableName(from: #"db.getCollection("stats").countDocuments({})"#) + #expect(plain == "user logs") + #expect(escaped == "say\"hi") + #expect(shadowed == "stats") + } + @Test("extractTableName returns nil when no FROM clause") func extractTableNameNoMatch() { #expect(QuerySqlParser.extractTableName(from: "SHOW TABLES") == nil) diff --git a/TableProTests/Core/Utilities/SQL/ColumnTypeSQLQuotingTests.swift b/TableProTests/Core/Utilities/SQL/ColumnTypeSQLQuotingTests.swift index 238f3c5f3..7bc893607 100644 --- a/TableProTests/Core/Utilities/SQL/ColumnTypeSQLQuotingTests.swift +++ b/TableProTests/Core/Utilities/SQL/ColumnTypeSQLQuotingTests.swift @@ -115,4 +115,31 @@ struct ColumnTypeSQLQuotingTests { #expect(lookup["id"] == .integer(rawType: "INT")) #expect(lookup["code"] == nil) } + + @Test("Character types are told apart from the other types the classifier files under text") + func characterTypeDetection() { + let character = [ + "VARCHAR(255)", "character varying", "text", "bpchar", "nvarchar2", "CLOB", "citext", "name", "String" + ] + for rawType in character { + #expect(ColumnTypeSQLQuoting.isCharacterType(.text(rawType: rawType)), "\(rawType)") + } + for rawType in ["uuid", "inet", "tsvector", "interval", "money", "unknown", "xml"] { + #expect(!ColumnTypeSQLQuoting.isCharacterType(.text(rawType: rawType)), "\(rawType)") + } + #expect(ColumnTypeSQLQuoting.isCharacterType(.text(rawType: nil))) + #expect(!ColumnTypeSQLQuoting.isCharacterType(.text(rawType: ""))) + #expect(!ColumnTypeSQLQuoting.isCharacterType(.integer(rawType: "int"))) + #expect(!ColumnTypeSQLQuoting.isCharacterType(.enumType(rawType: "ENUM(mood)", values: nil))) + #expect(!ColumnTypeSQLQuoting.isCharacterType(nil)) + } + + @Test("An array column is not text-like") + func arrayIsNotTextLike() { + let array = ColumnType.array(rawType: "text[]", element: .text(rawType: "text")) + #expect(!ColumnTypeSQLQuoting.isKnownTextLike(array)) + #expect(!ColumnTypeSQLQuoting.supportsEmptyStringComparison(array)) + #expect(ColumnTypeSQLQuoting.isKnownTextLike(.text(rawType: "text"))) + #expect(ColumnTypeSQLQuoting.isKnownTextLike(.set(rawType: "SET", values: nil))) + } } diff --git a/TableProTests/Plugins/MQLExportHelpersTests.swift b/TableProTests/Plugins/MQLExportHelpersTests.swift index c122c103d..8ed8c467c 100644 --- a/TableProTests/Plugins/MQLExportHelpersTests.swift +++ b/TableProTests/Plugins/MQLExportHelpersTests.swift @@ -122,4 +122,12 @@ struct MQLExportHelpersTests { #expect(MQLExportHelpers.mqlTextValue(for: "007", columnTypeName: "VARCHAR") == "\"007\"") #expect(MQLExportHelpers.mqlTextValue(for: "1.2.3", columnTypeName: "VARCHAR") == "\"1.2.3\"") } + + @Test("A collection is addressed as db. only when mongosh would resolve that to it") + func collectionAccessorAvoidsShadowedNames() { + #expect(MQLExportHelpers.collectionAccessor(for: "users") == "db.users") + #expect(MQLExportHelpers.collectionAccessor(for: "stats") == "db.getCollection(\"stats\")") + #expect(MQLExportHelpers.collectionAccessor(for: "my.data") == "db.getCollection(\"my.data\")") + #expect(MQLExportHelpers.collectionAccessor(for: "2024") == "db.getCollection(\"2024\")") + } } diff --git a/TableProTests/Plugins/MongoDBQueryBuilderTests.swift b/TableProTests/Plugins/MongoDBQueryBuilderTests.swift index eb828f98c..ca5aaf9e5 100644 --- a/TableProTests/Plugins/MongoDBQueryBuilderTests.swift +++ b/TableProTests/Plugins/MongoDBQueryBuilderTests.swift @@ -80,16 +80,16 @@ struct MongoDBQueryBuilderTests { #expect(!query.contains(".sort(")) } - @Test("Collection with special characters uses bracket notation") + @Test("Collection with special characters goes through getCollection") func collectionWithSpecialChars() { let query = builder.buildBaseQuery(collection: "my.collection") - #expect(query.hasPrefix("db[\"my.collection\"]")) + #expect(query.hasPrefix("db.getCollection(\"my.collection\")")) } - @Test("Collection starting with number uses bracket notation") + @Test("Collection starting with number goes through getCollection") func collectionStartingWithNumber() { let query = builder.buildBaseQuery(collection: "123abc") - #expect(query.hasPrefix("db[\"123abc\"]")) + #expect(query.hasPrefix("db.getCollection(\"123abc\")")) } @Test("Collection with simple name uses dot notation") @@ -495,7 +495,7 @@ struct MongoDBQueryBuilderTests { @Test("Count query with special collection name") func countQuerySpecialCollection() { let query = builder.buildCountQuery(collection: "my.data") - #expect(query.hasPrefix("db[\"my.data\"]")) + #expect(query.hasPrefix("db.getCollection(\"my.data\")")) #expect(query.contains(".countDocuments({})")) } @@ -507,16 +507,16 @@ struct MongoDBQueryBuilderTests { #expect(query == "db.users.find({})") } - @Test("Export query brackets a dotted collection name") + @Test("Export query reaches a dotted collection through getCollection") func exportQueryDottedCollection() { let query = builder.buildExportQuery(collection: "logs.2024.06") - #expect(query == "db[\"logs.2024.06\"].find({})") + #expect(query == "db.getCollection(\"logs.2024.06\").find({})") } @Test("Export query escapes quotes and backslashes in the collection name") func exportQueryEscapesCollectionName() { let query = builder.buildExportQuery(collection: "say\"hi\\bye") - #expect(query == "db[\"say\\\"hi\\\\bye\"].find({})") + #expect(query == "db.getCollection(\"say\\\"hi\\\\bye\").find({})") } @Test("Export query parses back to a find on the same collection") @@ -832,4 +832,78 @@ struct MongoDBQueryBuilderTests { #expect(!doc.contains("$binary")) } + // MARK: - Collection accessor + + @Test("A collection named after a db method is reached through getCollection") + func shadowedCollectionNamesUseGetCollection() { + for name in ["stats", "version", "toString", "constructor", "valueOf", "getName", "__proto__"] { + let query = builder.buildBaseQuery(collection: name) + #expect(query == "db.getCollection(\"\(name)\").find({}).limit(200)", "collection \(name)") + } + } + + @Test("A shadowed collection name parses back to a find on that collection") + func shadowedCollectionNameRoundTripsThroughTheParser() throws { + let operation = try MongoShellParser.parse(builder.buildBaseQuery(collection: "stats")) + guard case .find(let collection, _, _) = operation else { + Issue.record("Expected .find operation") + return + } + #expect(collection == "stats") + } + + @Test("Every method the shell puts on db is a name the accessor refuses to spell as db.") + func accessorCoversEveryDatabaseMethodOfTheShell() throws { + let regex = try NSRegularExpression(pattern: #"DB\.prototype\.([A-Za-z_][A-Za-z0-9_]*)\s*="#) + let source = MongoScriptPrelude.source + let matches = regex.matches(in: source, range: NSRange(source.startIndex..., in: source)) + let members = matches.compactMap { Range($0.range(at: 1), in: source).map { String(source[$0]) } } + #expect(members.count > 10) + for member in members { + #expect(MongoCollectionAccessor.isShadowedByDatabaseMember(member), "db.\(member) is a method") + } + } + + // MARK: - Raw filter normalization + + private static let normalizer = MongoDBRawFilterNormalizer() + + private var normalizingBuilder: MongoDBQueryBuilder { + MongoDBQueryBuilder(rawFilterNormalizer: { Self.normalizer.normalize($0) }) + } + + @Test("A raw filter in shell syntax is rewritten to Extended JSON for find and count alike") + func rawFilterInShellSyntaxBecomesExtendedJSON() throws { + let raw = PluginQueryFilter( + column: MongoDBQueryBuilder.rawFilterColumn, op: "RAW", + value: "{status: 'active', _id: ObjectId(\"507f1f77bcf86cd799439011\"), n: 3}" + ) + let doc = normalizingBuilder.buildFilterDocument(from: [raw]) + let expected = "{\"$and\": [{\"status\":\"active\"," + + "\"_id\":{\"$oid\":\"507f1f77bcf86cd799439011\"},\"n\":{\"$numberInt\":\"3\"}}]}" + #expect(doc == expected) + let parsed = try JSONSerialization.jsonObject(with: Data(doc.utf8)) as? [String: Any] + #expect((parsed?["$and"] as? [[String: Any]])?.count == 1) + let find = normalizingBuilder.buildFilteredQuery(collection: "users", queryFilters: [raw]) + #expect(find == "db.users.find(\(expected)).limit(200)") + } + + @Test("A raw filter the shell cannot evaluate is kept as typed") + func rawFilterThatFailsToEvaluateIsKeptVerbatim() { + let raw = PluginQueryFilter( + column: MongoDBQueryBuilder.rawFilterColumn, op: "RAW", value: "{status: }" + ) + let doc = normalizingBuilder.buildFilterDocument(from: [raw]) + #expect(doc == "{\"$and\": [{status: }]}") + } + + @Test("The normalizer serializes dates and regex literals the way the shell does") + func normalizerSerializesShellValues() { + let normalized = Self.normalizer.normalize("{at: ISODate(\"2024-01-02T00:00:00Z\"), name: /^bo/i}") + let expected = "{\"at\":{\"$date\":{\"$numberLong\":\"1704153600000\"}}," + + "\"name\":{\"$regularExpression\":{\"pattern\":\"^bo\",\"options\":\"i\"}}}" + #expect(normalized == expected) + #expect(Self.normalizer.normalize("{}") == "{}") + #expect(Self.normalizer.normalize("{_id: ObjectId()}") == nil) + } } diff --git a/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift b/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift index 0eadeb61d..d64035ca4 100644 --- a/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift +++ b/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift @@ -772,7 +772,7 @@ struct MongoDBStatementGeneratorTests { // MARK: - Collection Accessor - @Test("Collection with dots uses bracket notation") + @Test("Collection with dots goes through getCollection") func collectionBracketNotation() { let gen = MongoDBStatementGenerator( collectionName: "my.collection", @@ -794,7 +794,7 @@ struct MongoDBStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement.contains("db[\"my.collection\"]")) + #expect(results[0].statement.contains("db.getCollection(\"my.collection\")")) } // MARK: - Value Type Detection @@ -1046,7 +1046,7 @@ struct MongoDBStatementGeneratorTests { #expect(statements?.count == 1) let statement = statements?.first?.statement ?? "" - #expect(statement.hasPrefix("db[\"users\"].insertOne(")) + #expect(statement.hasPrefix("db.users.insertOne(")) let document = firstArgumentObject(in: statement) #expect((document?["_id"] as? [String: Any])?["$oid"] as? String == "507f1f77bcf86cd799439011") #expect(document?["name"] as? String == "Alice") diff --git a/project.yml b/project.yml index d176feb30..e05b001a8 100644 --- a/project.yml +++ b/project.yml @@ -401,6 +401,7 @@ targets: - Plugins/MongoDBDriverPlugin/MongoDBJsonNumber.swift - Plugins/MongoDBDriverPlugin/MongoDBNameValidator.swift - Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift + - Plugins/MongoDBDriverPlugin/MongoDBRawFilterNormalizer.swift - Plugins/MongoDBDriverPlugin/MongoDBSSLMapping.swift - Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift - Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift