From 465c68c1c59d473ddb8d2f4916d783d85796218e Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Mon, 20 Jul 2026 17:59:31 +0000 Subject: [PATCH 1/5] fix(kernel): render out-of-nanosecond-range TIMESTAMPs without wrapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel Rows path converted arrow timestamps with arrow's ToTime, which forms an int64-nanosecond intermediate (value * unit-multiplier). For a microsecond TIMESTAMP — Databricks' wire unit — that overflows int64 outside ~1678-2262 and silently wraps a valid instant to a wrong one (TIMESTAMP '0001-01-01' scanned back as 1754-08-30, '9999-12-31' as 1816-03-30). Convert via the unit-specific time.Unix/UnixMilli/UnixMicro constructors, which split into (seconds, nanoseconds) internally and represent the full 0001-9999 range Databricks TIMESTAMP allows. The default Thrift path is unaffected — it receives TIMESTAMP as a preformatted server string and never runs this multiply — so this closes a kernel-only divergence from Thrift. TestScanCellTimestampOutOfNanoRange pins both repro endpoints plus the int64-ns boundary years; the existing unit and location tests still pass. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/arrowscan/arrowscan.go | 28 ++++++++++++++++++++++- internal/arrowscan/arrowscan_test.go | 34 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/internal/arrowscan/arrowscan.go b/internal/arrowscan/arrowscan.go index cee634e2..2d70360a 100644 --- a/internal/arrowscan/arrowscan.go +++ b/internal/arrowscan/arrowscan.go @@ -173,7 +173,15 @@ func ScanCellCached(col arrow.Array, row int, loc *time.Location, keys *StructKe // it. The cross-backend parity test pins both sides to µs (matching the wire // reality), so it can't exercise a non-µs value; TestScanCellTimestampUnits // covers this arm's unit-correctness directly instead. - return inLocation(c.Value(row).ToTime(dt.Unit), loc), nil + // + // Convert via the unit-specific time.Unix* constructors rather than arrow's + // ToTime: ToTime forms an int64-nanosecond intermediate (value*multiplier), + // which overflows for TIMESTAMPs outside ~1678–2262 and silently wraps a valid + // instant to a wrong one (e.g. TIMESTAMP '0001-01-01' → 1754). time.UnixMicro/ + // UnixMilli/Unix split into (sec, nsec) internally, so the full 0001–9999 range + // round-trips. The Thrift default path never hits this — it receives TIMESTAMP + // as a preformatted server string — so this divergence is kernel-only. + return inLocation(timestampToTime(int64(c.Value(row)), dt.Unit), loc), nil case *array.Decimal128: dt := col.DataType().(*arrow.Decimal128Type) return decimalfmt.ExactString(c.Value(row), dt.Scale), nil @@ -276,6 +284,24 @@ func abs64(x int64) int64 { return x } +// timestampToTime converts an arrow timestamp value in the given unit to a UTC +// time.Time without arrow's ToTime int64-nanosecond overflow (see the *array.Timestamp +// scan arm). Each constructor takes its argument in native units and splits into +// (seconds, nanoseconds) internally, so the whole 0001–9999 range is representable; +// only the nanosecond arm passes ns directly (already the finest unit, cannot overflow). +func timestampToTime(v int64, unit arrow.TimeUnit) time.Time { + switch unit { + case arrow.Second: + return time.Unix(v, 0).UTC() + case arrow.Millisecond: + return time.UnixMilli(v).UTC() + case arrow.Microsecond: + return time.UnixMicro(v).UTC() + default: // arrow.Nanosecond + return time.Unix(0, v).UTC() + } +} + // inLocation renders t in loc, matching the Thrift path's .In(location); a nil // loc leaves the value in UTC (arrow's ToTime default). func inLocation(t time.Time, loc *time.Location) time.Time { diff --git a/internal/arrowscan/arrowscan_test.go b/internal/arrowscan/arrowscan_test.go index 957a8f47..f58740ef 100644 --- a/internal/arrowscan/arrowscan_test.go +++ b/internal/arrowscan/arrowscan_test.go @@ -274,6 +274,40 @@ func TestScanCellTimestampUnits(t *testing.T) { } } +// TestScanCellTimestampOutOfNanoRange pins the fix for the microsecond→nanosecond +// overflow: arrow's ToTime forms value*multiplier as an int64 ns count, which +// overflows for instants outside ~1678–2262 and silently wraps to a wrong time +// (TIMESTAMP '0001-01-01' scanned back as 1754, '9999-12-31' as 1816). The +// unit-split constructors used by ScanCell must round-trip the full 0001–9999 +// range that Databricks TIMESTAMP allows. +func TestScanCellTimestampOutOfNanoRange(t *testing.T) { + pool := memory.NewGoAllocator() + cases := []time.Time{ + time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), // TIMESTAMP '0001-01-01 00:00:00' + time.Date(9999, time.December, 31, 23, 59, 59, 0, time.UTC), // TIMESTAMP '9999-12-31 23:59:59' + time.Date(1677, time.January, 1, 0, 0, 0, 0, time.UTC), // just below the int64-ns floor + time.Date(2263, time.January, 1, 0, 0, 0, 0, time.UTC), // just above the int64-ns ceiling + } + for _, want := range cases { + t.Run(want.Format("2006-01-02"), func(t *testing.T) { + // Databricks TIMESTAMP is always microseconds on the wire. + b := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Microsecond}) + defer b.Release() + b.Append(arrow.Timestamp(want.UnixMicro())) + arr := b.NewArray() + defer arr.Release() + + v, err := ScanCell(arr, 0, nil) + if err != nil { + t.Fatal(err) + } + if got := v.(time.Time); !got.Equal(want) { + t.Errorf("got %v, want %v (arrow ToTime would have wrapped this)", got.UTC(), want) + } + }) + } +} + // ScanCell renders nested types (list/struct/map) to a JSON string matching the // Thrift path. func TestScanCellNested(t *testing.T) { From 21ad6fad8419527cabb0619d9089d0ea7685760a Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Mon, 20 Jul 2026 17:59:40 +0000 Subject: [PATCH 2/5] fix(kernel): report 0 affected rows for DDL to match Thrift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C ABI returns -1 for "not applicable / unknown" affected-row counts — DDL, SELECT, or a warehouse that doesn't surface the counter (the kernel's num_modified_rows Option None). The Thrift path reports 0 in those cases, so Result.RowsAffected() diverged: a CREATE/DROP SCHEMA returned -1 on the kernel backend and 0 on Thrift. Fold the -1 sentinel to 0 so RowsAffected() is identical across backends; a real DML count (>= 0) passes through unchanged. The rule lives in a pure, untagged normalizeAffectedRows so it is unit-testable in the default CGO_ENABLED=0 build alongside the other kernel decision helpers. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/backend/kernel/affectedrows.go | 21 +++++++++++++++++ internal/backend/kernel/affectedrows_test.go | 24 ++++++++++++++++++++ internal/backend/kernel/operation.go | 9 +++++++- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 internal/backend/kernel/affectedrows.go create mode 100644 internal/backend/kernel/affectedrows_test.go diff --git a/internal/backend/kernel/affectedrows.go b/internal/backend/kernel/affectedrows.go new file mode 100644 index 00000000..ca2bd564 --- /dev/null +++ b/internal/backend/kernel/affectedrows.go @@ -0,0 +1,21 @@ +package kernel + +// This file is intentionally NOT behind the `cgo && databricks_kernel` build tag. +// It holds the pure normalization of the C ABI's affected-row count so the rule is +// unit-testable in the default CGO_ENABLED=0 build (like paramBindArg in bindparams.go), +// separate from the cgo call site in operation.go. + +// normalizeAffectedRows maps the kernel C ABI's modified-row count onto the value +// database/sql's Result.RowsAffected() reports, matching the Thrift path. +// +// The C ABI returns -1 for "not applicable / unknown": DDL, SELECT, or a warehouse +// that doesn't surface the counter (the kernel's num_modified_rows Option is +// None). The Thrift path reports 0 in exactly those cases (TGetOperationStatusResp +// defaults NumModifiedRows to 0), so the -1 sentinel is folded to 0 for parity. +// A real DML count (>= 0) passes through unchanged. +func normalizeAffectedRows(n int64) int64 { + if n < 0 { + return 0 + } + return n +} diff --git a/internal/backend/kernel/affectedrows_test.go b/internal/backend/kernel/affectedrows_test.go new file mode 100644 index 00000000..9df1b5f0 --- /dev/null +++ b/internal/backend/kernel/affectedrows_test.go @@ -0,0 +1,24 @@ +package kernel + +import "testing" + +func TestNormalizeAffectedRows(t *testing.T) { + cases := []struct { + name string + in int64 + want int64 + }{ + {"ddl or select sentinel (-1) folds to Thrift's 0", -1, 0}, + {"any negative folds to 0", -42, 0}, + {"zero real count is preserved", 0, 0}, + {"positive DML count passes through", 7, 7}, + {"large DML count passes through", 1_000_000, 1_000_000}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeAffectedRows(tc.in); got != tc.want { + t.Errorf("normalizeAffectedRows(%d) = %d, want %d", tc.in, got, tc.want) + } + }) + } +} diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 4e0228b9..0bd82c58 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -194,7 +194,14 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b // Capture the modified-row count and server query id now, while exec is live — // the operation is closed (nulling exec) before these are read on the // ExecContext path, and the query-id pointer is only valid while exec lives. - op.affectedRows = int64(C.kernel_executed_statement_num_modified_rows(exec)) + // + // The C ABI returns -1 for "not applicable / unknown" (DDL, SELECT, or a + // warehouse that doesn't surface the counter — the kernel's Option None). + // The Thrift path reports 0 in that case (TGetOperationStatusResp defaults + // NumModifiedRows to 0), so normalize the sentinel to 0 to keep RowsAffected() + // identical across backends; real DML counts (>= 0) pass through unchanged. + // normalizeAffectedRows is the pure (CGO_ENABLED=0-testable) form of this rule. + op.affectedRows = normalizeAffectedRows(int64(C.kernel_executed_statement_num_modified_rows(exec))) // A nil execErr means the statement reached a terminal state server-side (it // committed). We deliberately do NOT re-check ctx.Err() here to convert a // completed statement into a cancellation: for a non-idempotent DML that would From c0772716302e4db232b4a0d016aec74bb0de962e Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 04:07:59 +0000 Subject: [PATCH 3/5] fix(kernel): forward User-Agent so query history attributes the Go driver The kernel backend never forwarded the driver's User-Agent, so the kernel's built-in UA leaked through and query history misattributed SEA-path queries. Forward the same composed UA the Thrift path sends via set_custom_header, so both backends are attributed alike. Default-on; WithUserAgentEntry still customizes it. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/backend/kernel/backend.go | 14 ++++++++++++++ internal/backend/kernel/config.go | 4 ++++ kernel_config.go | 3 +++ kernel_config_test.go | 4 +++- 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index febf37e4..5dedc4b3 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -132,6 +132,20 @@ func (k *KernelBackend) OpenSession(ctx context.Context) error { return err } + // User-Agent so query history attributes the kernel path to this driver. + if k.cfg.UserAgent != "" { + name := newCStr("User-Agent") + val := newCStr(k.cfg.UserAgent) + errSet := call(func() C.KernelStatusCode { + return C.kernel_session_config_set_custom_header(cfg, name.c, val.c) + }) + name.free() + val.free() + if errSet != nil { + return fmt.Errorf("kernel: set_custom_header[User-Agent]: %w", toConnError(errSet)) + } + } + // TLS: crypto/tls's InsecureSkipVerify accepts any server cert, so relax both // chain validation and the hostname check — mapping only one would leave the // kernel path stricter than the Thrift path it mirrors (a self-signed cert diff --git a/internal/backend/kernel/config.go b/internal/backend/kernel/config.go index 5727407c..e0577891 100644 --- a/internal/backend/kernel/config.go +++ b/internal/backend/kernel/config.go @@ -17,6 +17,10 @@ type Config struct { WarehouseID string // bare warehouse id; preferred over HTTPPath when set Auth Auth // PAT / OAuth M2M / OAuth U2M + // UserAgent is forwarded as the User-Agent header so the kernel path is + // attributed to this driver (not the kernel's built-in UA). Empty leaves it unset. + UserAgent string + // SessionConf carries server-bound session confs verbatim — the same map the // Thrift backend forwards (STATEMENT_TIMEOUT, QUERY_TAGS, TIMEZONE, …). SessionConf map[string]string diff --git a/kernel_config.go b/kernel_config.go index e4bf996f..72443729 100644 --- a/kernel_config.go +++ b/kernel_config.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/databricks-sql-go/auth/pat" dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/databricks/databricks-sql-go/internal/backend/kernel" + "github.com/databricks/databricks-sql-go/internal/client" "github.com/databricks/databricks-sql-go/internal/config" ) @@ -116,6 +117,8 @@ func buildKernelConfig(cfg *config.Config, kauth kernel.Auth) kernel.Config { WarehouseID: cfg.WarehouseID, Auth: kauth, Location: cfg.Location, + // Same UA the Thrift path sends, so query history attributes both alike. + UserAgent: client.BuildUserAgent(cfg), // Initial namespace: no kernel config setter, so the kernel backend applies // these post-connect via USE CATALOG / USE SCHEMA. Catalog: cfg.Catalog, diff --git a/kernel_config_test.go b/kernel_config_test.go index c152eeab..3a5a2de2 100644 --- a/kernel_config_test.go +++ b/kernel_config_test.go @@ -329,9 +329,11 @@ var kernelConfigFieldDisposition = map[string]string{ "MaxRows": "inert", "UseLz4Compression": "inert", // kernel negotiates compression internally + // Rides in the forwarded User-Agent header (set_custom_header). + "UserAgentEntry": "forwarded", + // Not applicable to the kernel path (Thrift/HTTP-transport or telemetry knobs // that don't reach the kernel binding). - "UserAgentEntry": "inert", // TODO: forward once the kernel exposes a UA setter "EnableTelemetry": "inert", "TelemetryBatchSize": "inert", "TelemetryFlushInterval": "inert", From 8815db9c35477a5db9bb546a43eb3a0054e000a9 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 18:16:27 +0000 Subject: [PATCH 4/5] fix(kernel): report VOID/NULL columns as STRING to match Thrift The server stringifies VOID over the Thrift wire (verified live: even bare SELECT NULL reports STRING_TYPE), same as intervals, but hands the kernel a native arrow.NULL schema. Map arrow.NULL to STRING so both backends report identical column metadata; correct the two parity tests that wrongly pinned NULL_TYPE (derived from the enum name, never captured live). Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/arrowscan/coltype.go | 77 ++++++--------------- internal/arrowscan/coltype_test.go | 22 +++--- internal/rows/coltype_thrift_parity_test.go | 37 ++++------ 3 files changed, 42 insertions(+), 94 deletions(-) diff --git a/internal/arrowscan/coltype.go b/internal/arrowscan/coltype.go index e795fcfc..7cdba4d7 100644 --- a/internal/arrowscan/coltype.go +++ b/internal/arrowscan/coltype.go @@ -10,32 +10,18 @@ import ( ) // ColumnTypeInfo is the per-column metadata database/sql surfaces through -// sql.ColumnType — the DatabaseTypeName, ScanType, and Length the optional -// driver.RowsColumnType* interfaces return. The kernel backend derives it from -// the result's Arrow schema; ColumnTypeInfoFor is the single mapping both the -// value scanner (ScanCellCached) and the type-metadata reporter agree on, so a -// column's reported type can never drift from what a row actually scans into. -// -// The mapping mirrors the Thrift backend (internal/rows/rows.go getScanType / -// ColumnTypeDatabaseTypeName / ColumnTypeLength) so a query reports byte-identical -// column metadata on either backend — the same "identical results across backends" -// contract the value renderers hold. The pure-Go guards are TestColumnTypeInfoFor, -// TestColumnTypeInfoScanTypeCoversScanner, and TestColumnTypeInfoMatchesThriftMapping -// (which cross-checks this mapping against the Thrift functions directly); the live -// TestKernelThriftColumnTypeParity confirms it against a real warehouse. +// sql.ColumnType. The kernel derives it from the result's Arrow schema via +// ColumnTypeInfoFor — the mapping the value scanner and type reporter share, kept +// byte-identical to the Thrift backend (guarded by the coltype parity tests). type ColumnTypeInfo struct { - // DatabaseTypeName is the Databricks type name (e.g. "BIGINT", "DECIMAL", - // "ARRAY"), matching what the Thrift path reports; "" for a type with no - // Databricks name. + // DatabaseTypeName is the Databricks type name (e.g. "BIGINT", "DECIMAL"), + // matching the Thrift path; "" for a type with no Databricks name. DatabaseTypeName string // ScanType is the Go type database/sql recommends scanning the column into, - // matching the Thrift path. nil (only for the NULL type) makes database/sql - // fall back to interface{}, exactly as the Thrift path's nil scan type does. + // matching the Thrift path. ScanType reflect.Type - // Length / HasLength report a variable-length column's length. As on the - // Thrift path, only variable-length types (string/binary/nested, and the - // server-stringified interval/geo types) report a length, and the reported - // value is math.MaxInt64 (unbounded); fixed-width types report (0, false). + // Length / HasLength report a variable-length column's unbounded length + // (math.MaxInt64), matching Thrift; fixed-width types report (0, false). Length int64 HasLength bool } @@ -58,14 +44,8 @@ var ( // ColumnTypeInfoFor maps an Arrow column type to the metadata database/sql // exposes, matching the Thrift backend for every Databricks type. The Arrow types -// listed here are exactly those ScanCellCached scans, so the type reported for a -// column and the value produced for its cells stay in lockstep. -// -// Notably: DECIMAL reports sql.RawBytes (the Thrift scan type for DECIMAL) even -// though the value is rendered as an exact string — matching Thrift, which also -// scans DECIMAL into RawBytes; and the interval / geo types report STRING because -// the Thrift server pre-formats them to strings and the kernel formats them -// Go-side to the same string, so both are indistinguishable to a caller. +// here are exactly those ScanCellCached scans, so a column's reported type and its +// scanned value stay in lockstep. func ColumnTypeInfoFor(dt arrow.DataType) ColumnTypeInfo { switch dt.ID() { case arrow.BOOL: @@ -79,11 +59,8 @@ func ColumnTypeInfoFor(dt arrow.DataType) ColumnTypeInfo { case arrow.INT64: return ColumnTypeInfo{DatabaseTypeName: "BIGINT", ScanType: scanTypeInt64} case arrow.UINT8, arrow.UINT16, arrow.UINT32, arrow.UINT64: - // Databricks SQL has no unsigned types, so these do not occur in practice; - // this arm is defensive and stays in lockstep with ScanCellCached, which - // widens every unsigned integer to int64 (driver.Value has no uint64). Report - // BIGINT/int64 to match that scanned value rather than falling through to the - // generic *interface{} default. + // Databricks SQL has no unsigned types; defensive arm matching ScanCellCached, + // which widens unsigned ints to int64 (driver.Value has no uint64). return ColumnTypeInfo{DatabaseTypeName: "BIGINT", ScanType: scanTypeInt64} case arrow.FLOAT32: return ColumnTypeInfo{DatabaseTypeName: "FLOAT", ScanType: scanTypeFloat32} @@ -100,10 +77,8 @@ func ColumnTypeInfoFor(dt arrow.DataType) ColumnTypeInfo { case arrow.TIMESTAMP: return ColumnTypeInfo{DatabaseTypeName: "TIMESTAMP", ScanType: scanTypeDateTime} case arrow.DECIMAL128: - // Thrift scans DECIMAL into sql.RawBytes; match it even though the kernel - // renders the value as an exact fixed-point string (both convert cleanly to - // a caller's *string/*[]byte, and reporting the same scan type keeps the - // backends indistinguishable). + // Match Thrift's sql.RawBytes scan type even though the kernel renders the + // value as an exact string — both convert cleanly to a caller's *string/*[]byte. return ColumnTypeInfo{DatabaseTypeName: "DECIMAL", ScanType: scanTypeRawBytes} case arrow.LIST, arrow.LARGE_LIST, arrow.FIXED_SIZE_LIST: return varLen("ARRAY", scanTypeRawBytes) @@ -112,24 +87,17 @@ func ColumnTypeInfoFor(dt arrow.DataType) ColumnTypeInfo { case arrow.STRUCT: return varLen("STRUCT", scanTypeRawBytes) case arrow.DURATION: - // INTERVAL DAY TO SECOND. Parity target is what the Thrift backend REPORTS, - // which is config-dependent: in the prod default (native-interval Arrow off) - // the server pre-formats intervals to text and declares the Thrift column - // STRING_TYPE, so Thrift's GetDBTypeName yields "STRING" and MaxInt64 length - // — verified live against both backends. We therefore report STRING here even - // though the kernel receives a native arrow.DURATION (which it formats Go-side - // to the identical string). If a warehouse ever enables native-interval Thrift - // the server would instead declare INTERVAL_DAY_TIME and Thrift would report - // that; matching STRING is correct for the default path the parity test pins, - // and the scanned VALUE is identical either way — only this label would differ. + // INTERVAL DAY TO SECOND: Thrift's prod default pre-formats intervals to text + // and declares STRING_TYPE (verified live), so match STRING; the kernel formats + // the native arrow.DURATION Go-side to the identical string. return varLen("STRING", scanTypeString) case arrow.INTERVAL_MONTHS: // INTERVAL YEAR TO MONTH — same server-config reasoning as arrow.DURATION. return varLen("STRING", scanTypeString) case arrow.NULL: - // The NULL type has no scan type on the Thrift path (nil → database/sql - // falls back to interface{}); mirror that. - return ColumnTypeInfo{DatabaseTypeName: "NULL", ScanType: nil} + // VOID/NULL columns: the server stringifies them over Thrift (verified live: + // even bare SELECT NULL reports STRING), same as intervals — so match STRING. + return varLen("STRING", scanTypeString) default: // A type ScanCellCached does not handle: report the Thrift default scan type // (*interface{}) and no database name, rather than inventing one. @@ -137,9 +105,8 @@ func ColumnTypeInfoFor(dt arrow.DataType) ColumnTypeInfo { } } -// varLen builds a ColumnTypeInfo for a variable-length type, which reports an -// unbounded length (math.MaxInt64) just as the Thrift path does for -// string/binary/nested columns. +// varLen builds a ColumnTypeInfo for a variable-length type, reporting the +// unbounded length (math.MaxInt64) the Thrift path uses for such columns. func varLen(name string, scan reflect.Type) ColumnTypeInfo { return ColumnTypeInfo{DatabaseTypeName: name, ScanType: scan, Length: math.MaxInt64, HasLength: true} } diff --git a/internal/arrowscan/coltype_test.go b/internal/arrowscan/coltype_test.go index ac71ad0c..3878f79f 100644 --- a/internal/arrowscan/coltype_test.go +++ b/internal/arrowscan/coltype_test.go @@ -10,13 +10,10 @@ import ( "github.com/apache/arrow/go/v12/arrow" ) -// TestColumnTypeInfoFor pins the Arrow → column-metadata mapping the kernel -// backend reports through sql.ColumnType, so it stays byte-identical to the -// Thrift path (internal/rows/rows.go). This is a pure-Go, no-warehouse guard: the -// gap it regresses (PECOBLR-3692) was invisible to the value-parity suites -// because they compare scanned VALUES, not Rows.ColumnType* metadata. The -// expected DatabaseTypeName / ScanType / Length values are the ground truth -// captured from the Thrift backend on a live warehouse. +// TestColumnTypeInfoFor pins the Arrow → column-metadata mapping so it stays +// byte-identical to the Thrift path — a pure-Go guard the value-parity suites miss +// (they compare scanned values, not ColumnType metadata). Expected values are +// ground truth captured from Thrift on a live warehouse. func TestColumnTypeInfoFor(t *testing.T) { str := reflect.TypeOf("") raw := reflect.TypeOf(sql.RawBytes{}) @@ -53,7 +50,8 @@ func TestColumnTypeInfoFor(t *testing.T) { {"struct", arrow.StructOf(arrow.Field{Name: "x", Type: arrow.PrimitiveTypes.Int64}), "STRUCT", raw, math.MaxInt64, true}, {"duration", &arrow.DurationType{Unit: arrow.Microsecond}, "STRING", str, math.MaxInt64, true}, {"month_interval", arrow.FixedWidthTypes.MonthInterval, "STRING", str, math.MaxInt64, true}, - {"null", arrow.Null, "NULL", nil, 0, false}, + // VOID/NULL: Thrift stringifies to STRING on the wire (verified live), like intervals. + {"null", arrow.Null, "STRING", str, math.MaxInt64, true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -72,12 +70,8 @@ func TestColumnTypeInfoFor(t *testing.T) { } // TestColumnTypeInfoScanTypeCoversScanner is the lockstep guard: every Arrow type -// the value scanner (ScanCellCached) handles must also have a non-fallback entry -// in ColumnTypeInfoFor, so a future scalar type added to the scanner without a -// matching type-metadata entry is caught here rather than silently reporting the -// generic *interface{} scan type at runtime. It checks the representative arrays -// the scanner switches on; the NULL type is intentionally excluded (its nil scan -// type is the correct, Thrift-matching value). +// ScanCellCached handles must have a non-fallback ColumnTypeInfoFor entry, so a new +// scanner type without matching metadata is caught here, not at runtime. func TestColumnTypeInfoScanTypeCoversScanner(t *testing.T) { unknown := reflect.TypeOf(new(any)) scannerTypes := []arrow.DataType{ diff --git a/internal/rows/coltype_thrift_parity_test.go b/internal/rows/coltype_thrift_parity_test.go index be178a64..7635ec00 100644 --- a/internal/rows/coltype_thrift_parity_test.go +++ b/internal/rows/coltype_thrift_parity_test.go @@ -10,24 +10,12 @@ import ( "github.com/databricks/databricks-sql-go/internal/rows/rowscanner" ) -// TestColumnTypeInfoMatchesThriftMapping is the pure-Go drift guard the kernel -// backend's column-metadata mapping (internal/arrowscan.ColumnTypeInfoFor) was -// missing. That mapping restates the Thrift path's per-type decisions — getScanType -// (this package), rowscanner.GetDBTypeName, and ColumnTypeLength — but pinned its own -// hardcoded expectations, so a change to the Thrift mapping (e.g. DECIMAL's scan -// type) would leave arrowscan silently divergent while CI stayed green; parity was -// then enforced only by the warehouse-gated, build-tagged TestKernelThriftColumnTypeParity -// (nightly, not PR CI). -// -// This test cross-checks the two mappings DIRECTLY, with no cgo and no warehouse, so -// drift fails a default CGO_ENABLED=0 PR run. The two switch on different type systems -// (Arrow IDs vs Thrift TTypeIds), so it can't share a map — instead it enumerates the -// shared Databricks types as {Arrow type, equivalent Thrift TColumnDesc} pairs and -// asserts ColumnTypeInfoFor(arrow) reports exactly what the Thrift functions produce -// for the paired column. -// -// It lives in package rows (white-box) to reach the unexported getScanType; -// arrowscan is a pure leaf import (no cycle). +// TestColumnTypeInfoMatchesThriftMapping cross-checks arrowscan.ColumnTypeInfoFor +// against the Thrift mapping (getScanType / GetDBTypeName / ColumnTypeLength) +// directly, so a change to one that isn't mirrored in the other fails a pure-Go PR +// run — not just the nightly warehouse-gated parity test. It enumerates each shared +// type as an {Arrow type, equivalent Thrift TColumnDesc} pair and asserts they agree. +// White-box (package rows) to reach the unexported getScanType. func TestColumnTypeInfoMatchesThriftMapping(t *testing.T) { // thriftCol builds the minimal TColumnDesc the Thrift mapping functions read. thriftCol := func(id cli_service.TTypeId) *cli_service.TColumnDesc { @@ -63,15 +51,14 @@ func TestColumnTypeInfoMatchesThriftMapping(t *testing.T) { {"array", arrow.ListOf(arrow.PrimitiveTypes.Int64), cli_service.TTypeId_ARRAY_TYPE}, {"map", arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64), cli_service.TTypeId_MAP_TYPE}, {"struct", arrow.StructOf(arrow.Field{Name: "x", Type: arrow.PrimitiveTypes.Int64}), cli_service.TTypeId_STRUCT_TYPE}, - // Interval types: in the prod default (native-interval Arrow off) the server - // pre-formats intervals to text and declares the column STRING_TYPE, so the - // kernel — which receives native arrow.DURATION / arrow.INTERVAL_MONTHS and - // formats them Go-side to the same string — must report the STRING metadata the - // Thrift path reports for STRING_TYPE. Pairing them with STRING_TYPE here pins - // exactly that (see the arrowscan DURATION / INTERVAL_MONTHS arms). + // Intervals: the server pre-formats them to text and declares STRING_TYPE over + // Thrift, so pair with STRING_TYPE (the kernel formats the native arrow value to + // the same string). See the arrowscan DURATION / INTERVAL_MONTHS arms. {"interval_day_time", arrow.FixedWidthTypes.Duration_us, cli_service.TTypeId_STRING_TYPE}, {"interval_year_month", arrow.FixedWidthTypes.MonthInterval, cli_service.TTypeId_STRING_TYPE}, - {"null", arrow.Null, cli_service.TTypeId_NULL_TYPE}, + // VOID/NULL: the server stringifies it over Thrift as STRING_TYPE (verified + // live — even bare SELECT NULL), same as the interval types above. + {"null", arrow.Null, cli_service.TTypeId_STRING_TYPE}, } for _, c := range cases { From 0163bcdcd14c01b9145e053fad1d16ed1a746ad3 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 20:38:48 +0000 Subject: [PATCH 5/5] test(kernel): cover UA forwarding + VOID column-type parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review of the parity fixes: assert kc.UserAgent == BuildUserAgent (non-empty) in TestBuildKernelConfig so deleting the forwarding or an empty UA fails a pure-Go run; add a CAST(NULL AS VOID) column to the live column-type parity query so the arrow.NULL→STRING mapping is guarded against the real server, not only the pure-Go tests. Trim the duplicated -1→0 rationale at the affected-rows call site to a one-liner. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/backend/kernel/operation.go | 8 +------- kernel_config_test.go | 6 ++++++ kernel_parity_test.go | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 0bd82c58..cc60a5bc 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -194,13 +194,7 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b // Capture the modified-row count and server query id now, while exec is live — // the operation is closed (nulling exec) before these are read on the // ExecContext path, and the query-id pointer is only valid while exec lives. - // - // The C ABI returns -1 for "not applicable / unknown" (DDL, SELECT, or a - // warehouse that doesn't surface the counter — the kernel's Option None). - // The Thrift path reports 0 in that case (TGetOperationStatusResp defaults - // NumModifiedRows to 0), so normalize the sentinel to 0 to keep RowsAffected() - // identical across backends; real DML counts (>= 0) pass through unchanged. - // normalizeAffectedRows is the pure (CGO_ENABLED=0-testable) form of this rule. + // normalizeAffectedRows folds the C ABI's -1 sentinel to 0 for Thrift parity. op.affectedRows = normalizeAffectedRows(int64(C.kernel_executed_statement_num_modified_rows(exec))) // A nil execErr means the statement reached a terminal state server-side (it // committed). We deliberately do NOT re-check ctx.Err() here to convert a diff --git a/kernel_config_test.go b/kernel_config_test.go index 3a5a2de2..c2795bf4 100644 --- a/kernel_config_test.go +++ b/kernel_config_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/databricks-sql-go/auth/pat" dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/databricks/databricks-sql-go/internal/backend/kernel" + "github.com/databricks/databricks-sql-go/internal/client" "github.com/databricks/databricks-sql-go/internal/config" ) @@ -443,6 +444,11 @@ func TestBuildKernelConfig(t *testing.T) { if kc.Auth.Mode != kauth.Mode || kc.Auth.Token != kauth.Token { t.Errorf("auth not forwarded: got %+v, want %+v", kc.Auth, kauth) } + // UserAgent must be the driver's composed UA, non-empty — else query + // history mis-attributes SEA-path queries to the kernel's built-in UA. + if want := client.BuildUserAgent(c); kc.UserAgent == "" || kc.UserAgent != want { + t.Errorf("UserAgent not forwarded: got %q, want %q", kc.UserAgent, want) + } }) t.Run("MaxChunksInMemory injected into kernel SessionConf", func(t *testing.T) { diff --git a/kernel_parity_test.go b/kernel_parity_test.go index 4987e152..ed9e52ed 100644 --- a/kernel_parity_test.go +++ b/kernel_parity_test.go @@ -116,7 +116,9 @@ func TestKernelThriftColumnTypeParity(t *testing.T) { "CAST('2020-01-01 00:00:00' AS TIMESTAMP) a_ts, CAST('abc' AS BINARY) a_binary, " + "array(1,2,3) a_array, map('k',1) a_map, named_struct('x',1) a_struct, " + `parse_json('{"a":1}') a_variant, INTERVAL '1' DAY a_iv_dt, ` + - "INTERVAL '1' MONTH a_iv_ym, st_point(1,2) a_geom" + // VOID: the server declares it STRING_TYPE over Thrift; asserts the kernel + // arrow.NULL→STRING mapping matches live, not just in the pure-Go tests. + "INTERVAL '1' MONTH a_iv_ym, st_point(1,2) a_geom, CAST(NULL AS VOID) a_void" kernelDB := kernelTestDB(t) defer kernelDB.Close()