Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions apps/docs/content/docs/en/integrations/snowflake.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ Insert structured JSON rows using bound values.
| `database` | string | Yes | Database name |
| `schema` | string | Yes | Schema name |
| `table` | string | Yes | Target Snowflake table name within the selected database and schema context |
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest. |
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead. |

#### Output

Expand Down Expand Up @@ -210,7 +210,7 @@ Update matching rows with a bound MERGE statement without inserting new rows.
| `database` | string | Yes | Database name |
| `schema` | string | Yes | Schema name |
| `table` | string | Yes | Target Snowflake table name within the selected database and schema context |
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest. |
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead. |
| `matchColumns` | array | Yes | Columns used to match target rows. Match values must be non-null and unique across the submitted rows. |

#### Output
Expand Down Expand Up @@ -257,7 +257,7 @@ Update matching rows and insert unmatched rows with a bound MERGE statement.
| `database` | string | Yes | Database name |
| `schema` | string | Yes | Schema name |
| `table` | string | Yes | Target Snowflake table name within the selected database and schema context |
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest. |
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead. |
| `matchColumns` | array | Yes | Columns used to match target rows. Match values must be non-null and unique across the submitted rows. |

#### Output
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/blocks/blocks/snowflake.ts
Original file line number Diff line number Diff line change
Expand Up @@ -649,7 +649,7 @@ export const SnowflakeBlock: BlockConfig<SnowflakeStatementResponse> = {
rows: {
type: 'string',
description:
'Structured rows as a JSON array. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
'Structured rows as a JSON array. For bulk loads, stage the files and use Load Data instead.',
},
matchColumns: {
type: 'string',
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/tools/generated/tool-metadata.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion apps/sim/tools/snowflake/insert_rows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ export const insertRowsTool: ToolConfig<SnowflakeInsertRowsParams, SnowflakeStat
required: true,
visibility: 'user-or-llm',
description:
'Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
'Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead.',
},
},
request: snowflakeStatementRequest((params) =>
Expand Down
39 changes: 0 additions & 39 deletions apps/sim/tools/snowflake/sql.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,45 +172,6 @@ describe('Snowflake SQL builders', () => {
)
})

it('caps the number of rows per structured write', () => {
const rows = Array.from({ length: 1001 }, (_, index) => ({ id: index }))
expect(() => buildInsertRows({ ...table, rows })).toThrow('cannot exceed 1000 per call')
expect(() => buildInsertRows({ ...table, rows })).toThrow('snowflake_load_data')
expect(() => buildInsertRows({ ...table, rows: [{ blob: 'x'.repeat(1_000_001) }] })).toThrow(
'statement budget'
)
})

it('counts the bound value budget in UTF-8 bytes, not UTF-16 code units', () => {
expect(() => buildInsertRows({ ...table, rows: [{ blob: '中'.repeat(999_999) }] })).toThrow(
'statement budget'
)
expect(() =>
buildInsertRows({ ...table, rows: [{ blob: '中'.repeat(333_333) }] })
).not.toThrow()
})

it('applies the same statement budget to explicit bindings', () => {
expect(() =>
normalizeBindings({ '1': { type: 'TEXT', value: 'x'.repeat(1_000_001) } })
).toThrow('statement budget')
expect(() =>
normalizeBindings({
'1': { type: 'TEXT', value: 'x'.repeat(600_000) },
'2': { type: 'TEXT', value: 'x'.repeat(600_000) },
})
).toThrow('statement budget')
expect(() =>
buildCallProcedure({
...context,
database: 'ANALYTICS',
schema: 'PUBLIC',
procedureName: 'REFRESH_MODEL',
procedureArguments: [{ type: 'TEXT', value: '中'.repeat(999_999) }],
})
).toThrow('statement budget')
})

it('routes a mixed semi-structured column through a single whole-column PARSE_JSON', () => {
const result = buildInsertRows({
...table,
Expand Down
37 changes: 0 additions & 37 deletions apps/sim/tools/snowflake/sql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,26 +90,6 @@ function requireQueryId(queryId: string): string {
return trimmed
}

/**
* Snowflake recommends limiting query text to 1 MB per statement, and that limit explicitly covers
* values supplied through bindings. Statements above it still execute but are truncated before the
* metadata store persists them, so they can no longer be retried or inspected.
*
* The budget is a byte budget, so it is measured against the UTF-8 encoding rather than the
* JavaScript string length — a multi-byte string is up to 3x longer on the wire than in code units.
*/
const MAX_BOUND_VALUE_BYTES = 1_000_000

const BULK_INGEST_HINT = 'stage the data and use snowflake_load_data for bulk ingest'

function assertBoundBytesWithinBudget(boundBytes: number): void {
if (boundBytes > MAX_BOUND_VALUE_BYTES) {
throw new Error(
`Snowflake bound values exceed the ${MAX_BOUND_VALUE_BYTES} byte statement budget; send fewer rows per call or ${BULK_INGEST_HINT}`
)
}
}

export function normalizeBindings(
input?: Record<string, SnowflakeBinding>
): Record<string, SnowflakeBinding> | undefined {
Expand All @@ -119,7 +99,6 @@ export function normalizeBindings(
}
const normalized: Record<string, SnowflakeBinding> = {}
let hasBindings = false
let boundBytes = 0
for (const position in input) {
if (!Object.hasOwn(input, position)) continue
hasBindings = true
Expand All @@ -136,27 +115,16 @@ export function normalizeBindings(
if (typeof binding.value !== 'string') {
throw new Error(`binding ${position} value must be a string`)
}
boundBytes += Buffer.byteLength(binding.value, 'utf8')
assertBoundBytesWithinBudget(boundBytes)
normalized[position] = { type: binding.type, value: binding.value }
}
return hasBindings ? normalized : undefined
}

/**
* Structured writes build a single statement holding every row, so the row count is capped well
* below the byte budget to keep a typical write far away from the 1 MB statement recommendation.
*/
const MAX_WRITE_ROWS = 1_000

class BindingsBuilder {
readonly bindings: Record<string, SnowflakeBinding> = {}
private position = 0
private boundBytes = 0

private addBinding(type: SnowflakeBinding['type'], value: string): string {
this.boundBytes += Buffer.byteLength(value, 'utf8')
assertBoundBytesWithinBudget(this.boundBytes)
this.position += 1
const key = String(this.position)
this.bindings[key] = { type, value }
Expand Down Expand Up @@ -195,11 +163,6 @@ class BindingsBuilder {

function validateRows(rows: Array<Record<string, unknown>>): string[] {
if (!Array.isArray(rows) || rows.length === 0) throw new Error('rows must be a non-empty array')
if (rows.length > MAX_WRITE_ROWS) {
throw new Error(
`rows cannot exceed ${MAX_WRITE_ROWS} per call; send the rows in smaller batches or ${BULK_INGEST_HINT}`
)
}
const columns = Object.keys(rows[0] ?? {})
if (columns.length === 0) throw new Error('rows must contain at least one column')
const identifierKeys = new Set<string>()
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/tools/snowflake/update_rows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ export const updateRowsTool: ToolConfig<SnowflakeUpdateRowsParams, SnowflakeStat
required: true,
visibility: 'user-or-llm',
description:
'Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
'Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead.',
},
matchColumns: {
type: 'array',
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/tools/snowflake/upsert_rows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ export const upsertRowsTool: ToolConfig<SnowflakeUpsertRowsParams, SnowflakeStat
required: true,
visibility: 'user-or-llm',
description:
'Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
'Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead.',
},
matchColumns: {
type: 'array',
Expand Down
Loading