diff --git a/_data/shared_chrome.json b/_data/shared_chrome.json index d5492d5700..b4c1b8f118 100644 --- a/_data/shared_chrome.json +++ b/_data/shared_chrome.json @@ -1,7 +1,7 @@ { "$schema": "https://www.metabase.com/shared/chrome.schema.json", "version": 1, - "generated_at": "2026-07-23T20:32:22+00:00", + "generated_at": "2026-07-24T20:05:03+00:00", "origin": "https://www.metabase.com", "stylesheets": [ { diff --git a/_docs/latest/configuring-metabase/config-template.md b/_docs/latest/configuring-metabase/config-template.md index 6a1f6ff6ee..6bfd9928e6 100644 --- a/_docs/latest/configuring-metabase/config-template.md +++ b/_docs/latest/configuring-metabase/config-template.md @@ -219,8 +219,26 @@ config: ldap-user-filter: (&(objectClass=inetOrgPerson)(|(uid={login})(mail={login}))) ldap-user-provisioning-enabled: true license-token-missing-banner-dismissal-timestamp: [] + llm-anthropic-api-base-url: https://api.anthropic.com llm-anthropic-api-key: null + llm-anthropic-model: claude-opus-4-5-20251101 + llm-azure-api-base-url: null + llm-azure-api-key: null + llm-bedrock-access-key-id: null + llm-bedrock-region: us-east-1 + llm-bedrock-secret-access-key: null + llm-bedrock-session-token: null + llm-connection-timeout-ms: 5000 + llm-max-tokens: 4096 llm-metabot-provider: anthropic/claude-sonnet-4-6 + llm-openai-api-base-url: https://api.openai.com + llm-openai-api-key: null + llm-openai-model: gpt-5.4 + llm-openrouter-api-base-url: https://openrouter.ai/api + llm-openrouter-api-key: null + llm-rate-limit-per-ip: 100 + llm-rate-limit-per-user: 20 + llm-request-timeout-ms: 60000 load-analytics-content: true loading-message: doing-science login-page-illustration: default diff --git a/_docs/latest/configuring-metabase/environment-variables.md b/_docs/latest/configuring-metabase/environment-variables.md index 0f39372d38..c37188ef29 100644 --- a/_docs/latest/configuring-metabase/environment-variables.md +++ b/_docs/latest/configuring-metabase/environment-variables.md @@ -1296,6 +1296,14 @@ When set to `true`, users who log in via LDAP will automatically get a Metabase The array of last two ISO8601 dates when an admin dismissed the license token missing banner. +### `MB_LLM_ANTHROPIC_API_BASE_URL` + +- Type: string +- Default: `https://api.anthropic.com` +- [Configuration file name](./config-file): `llm-anthropic-api-base-url` + +The Anthropic API base URL. + ### `MB_LLM_ANTHROPIC_API_KEY` - Type: string @@ -1304,6 +1312,78 @@ The array of last two ISO8601 dates when an admin dismissed the license token mi The Anthropic API Key. +### `MB_LLM_ANTHROPIC_MODEL` + +- Type: string +- Default: `claude-opus-4-5-20251101` +- [Configuration file name](./config-file): `llm-anthropic-model` + +The Anthropic model to use. + +### `MB_LLM_AZURE_API_BASE_URL` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-azure-api-base-url` + +The base URL of the Azure resource's OpenAI- or Anthropic-compatible surface, e.g. `https://.services.ai.azure.com/openai`. + +### `MB_LLM_AZURE_API_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-azure-api-key` + +The API key for the Azure resource hosting your models. + +### `MB_LLM_BEDROCK_ACCESS_KEY_ID` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-bedrock-access-key-id` + +The AWS Access Key ID for Amazon Bedrock. + +### `MB_LLM_BEDROCK_REGION` + +- Type: string +- Default: `us-east-1` +- [Configuration file name](./config-file): `llm-bedrock-region` + +The AWS region for Amazon Bedrock (e.g. us-east-1). + +### `MB_LLM_BEDROCK_SECRET_ACCESS_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-bedrock-secret-access-key` + +The AWS Secret Access Key for Amazon Bedrock. + +### `MB_LLM_BEDROCK_SESSION_TOKEN` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-bedrock-session-token` + +The AWS Session Token for Amazon Bedrock. Only needed for temporary credentials. + +### `MB_LLM_CONNECTION_TIMEOUT_MS` + +- Type: integer +- Default: `5000` +- [Configuration file name](./config-file): `llm-connection-timeout-ms` + +Connection timeout in milliseconds for LLM API requests. + +### `MB_LLM_MAX_TOKENS` + +- Type: integer +- Default: `4096` +- [Configuration file name](./config-file): `llm-max-tokens` + +Maximum tokens for LLM responses. + ### `MB_LLM_METABOT_PROVIDER` - Type: string @@ -1312,6 +1392,70 @@ The Anthropic API Key. The AI provider and model for Metabot. Format: provider/model-name, e.g. `anthropic/claude-haiku-4-5`, `openai/gpt-5.4`, `openrouter/anthropic/claude-haiku-4.5`. +### `MB_LLM_OPENAI_API_BASE_URL` + +- Type: string +- Default: `https://api.openai.com` +- [Configuration file name](./config-file): `llm-openai-api-base-url` + +The OpenAI API base URL. + +### `MB_LLM_OPENAI_API_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-openai-api-key` + +The OpenAI API Key. + +### `MB_LLM_OPENAI_MODEL` + +- Type: string +- Default: `gpt-5.4` +- [Configuration file name](./config-file): `llm-openai-model` + +The OpenAI Model (e.g. 'gpt-5.5', 'gpt-5.4-mini'). + +### `MB_LLM_OPENROUTER_API_BASE_URL` + +- Type: string +- Default: `https://openrouter.ai/api` +- [Configuration file name](./config-file): `llm-openrouter-api-base-url` + +The OpenRouter API base URL used for Chat Completions. + +### `MB_LLM_OPENROUTER_API_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-openrouter-api-key` + +The OpenRouter API Key. + +### `MB_LLM_RATE_LIMIT_PER_IP` + +- Type: integer +- Default: `100` +- [Configuration file name](./config-file): `llm-rate-limit-per-ip` + +Maximum SQL generation requests per IP address per minute. + +### `MB_LLM_RATE_LIMIT_PER_USER` + +- Type: integer +- Default: `20` +- [Configuration file name](./config-file): `llm-rate-limit-per-user` + +Maximum SQL generation requests per user per minute. + +### `MB_LLM_REQUEST_TIMEOUT_MS` + +- Type: integer +- Default: `60000` +- [Configuration file name](./config-file): `llm-request-timeout-ms` + +Socket timeout in milliseconds for LLM API requests. + ### `MB_LOAD_ANALYTICS_CONTENT` - Type: boolean diff --git a/_docs/latest/data-studio/transforms/jobs-and-runs.md b/_docs/latest/data-studio/transforms/jobs-and-runs.md index 8b2d800ba9..2ffdedd545 100644 --- a/_docs/latest/data-studio/transforms/jobs-and-runs.md +++ b/_docs/latest/data-studio/transforms/jobs-and-runs.md @@ -47,7 +47,7 @@ Jobs have two components: schedule and tags. - **Schedule** determines when the job will be executed: daily, hourly, etc. You can specify a custom cron schedule (e.g. "Every weekday at 9:05 AM"). The times are given in your Metabase's system timezone. - **Tags** determine _which_ transforms a job runs, not when the job runs. For example, you can create a `Weekdays` tag, add that tag to a few transforms, then create a job that runs all the transforms with the `Weekdays` tag every weekday at 9:05AM. -Jobs will run all transforms tagged with any of the tags, plus any transforms that the tagged transforms depend on, see [Jobs will run all dependent transforms](#jobs-will-run-all-dependent-transforms). +Jobs will run all transforms tagged with any of the tags, plus any transforms that the tagged transforms depend on that aren't already up to date, see [Jobs include all dependent transforms](#jobs-include-all-dependent-transforms). You can see which transforms a job will run and in which order on the job's page. @@ -113,13 +113,22 @@ To delete a job: 2. Find the job you want to delete and click the **three dots** icon to the right of the job's name. 3. Select **Delete**. -## Jobs will run all dependent transforms +## Jobs include all dependent transforms -If one transform depends on another, Metabase will run the dependency first, even if that transform isn't tagged in the job. So if transform B depends on A, Metabase will first run A, even if A doesn't have a tag. +If one transform depends on another, Metabase adds the dependency to the job and runs it first, even if that transform isn't tagged in the job. So if transform B depends on A, Metabase makes sure A is up to date before running B. -This means that you can explicitly tag transform A to run daily, and transform B hourly, but because transform B depends on transform A, transform A will _also_ run hourly (in addition to daily), despite not having the tag. +## Jobs skip dependencies that are already up to date -You can see which transforms a job will run (and in which order) on the job's page in **Data Studio > Jobs**. +Metabase won't re-run a dependency that's still fresh. That way, a transform pulled into a frequent job doesn't get rebuilt more often than its own schedule calls for. So, for example, if you tag transform A `daily`, and transform B `hourly`, and hourly B depends on daily A, then A runs on its own daily schedule. The hourly job that runs B won't rerun A every hour. + +Metabase skips a dependency when the dependency: + +- Has its own tags and jobs, and none of those jobs' schedules have come due since the dependency last ran successfully. +- Has no tags at all, and it's already run successfully at least once. + +## See which transforms a job will run + +The job's page in **Data Studio > Jobs** lists every transform the job will run and in which order. The **Notes** column tells you when Metabase will skip a dependency, and flags dependencies that have no schedule of their own. ## Runs @@ -129,4 +138,11 @@ You can see all past and current transform runs (both manual and scheduled) by g You can click on any transform run to see more details about the run, like the error logs. To go to the transform definition from the transform run page, click on the icon next the transform name in the right sidebar. -The "Tags" column in the **Runs** table will only show the transform's specific tags. But the run might not have anything to do with those tags. Another job with different tags could have run the transform because [jobs will run _all_ dependent transforms](#jobs-will-run-all-dependent-transforms). +The "Tags" column in the **Runs** table will only show the transform's specific tags. But the run might not have anything to do with those tags. Another job with different tags could have run the transform because [jobs include all dependent transforms](#jobs-include-all-dependent-transforms). + +## Emails about failed transforms + +If your Metabase is [set up to send email](../../configuring-metabase/email), Metabase will let people know when transforms fail. + +- **Individual transform failures**: when transforms fail during a scheduled job run, Metabase emails the last person to edit each failed transform (or the transform's creator, or admins if neither is active). Each email covers a single job run. +- **Daily digest of job failures**: each morning, Metabase emails all admins a summary of the scheduled job runs that failed or timed out the previous day. Manual runs aren't included in the digest, and Metabase skips the email entirely if no scheduled runs failed. diff --git a/_docs/latest/data-studio/transforms/python-transforms.md b/_docs/latest/data-studio/transforms/python-transforms.md index ca58e10366..dab27ed857 100644 --- a/_docs/latest/data-studio/transforms/python-transforms.md +++ b/_docs/latest/data-studio/transforms/python-transforms.md @@ -119,7 +119,7 @@ To use functions or classes from your Python library: ## Incremental Python transforms -By default, Metabase will process all the data in all input tables, drop the existing target table (if one exists), and create a new table with the processed data. You can tell Metabase to only write **new** data to your target table by marking your transform as incremental. +By default, Metabase will process all the data in all input tables, drop the existing target table (if one exists), and create a new table with the processed data. You can tell Metabase to only process **new** data by marking your transform as incremental. ### Prerequisites for incremental transforms @@ -131,7 +131,8 @@ To make a Python transform incremental: 1. Go to the transform's page in **Data studio > Transforms**. 2. Switch to **Settings** tab. -3. In **Column to check for new values**, select the column in one of the source tables that Metabase should check to determine which values are new. Only some columns are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +3. In **Field to check for new values**, select the field in one of the source tables that Metabase should scan to determine which records are new or changed. Only some fields are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +4. (Optional) To update matching rows instead of appending new ones, [add a merge key](./transforms-overview#add-merge-keys-to-upsert-rows). ## Current limitations of Python transforms diff --git a/_docs/latest/data-studio/transforms/query-transforms.md b/_docs/latest/data-studio/transforms/query-transforms.md index b0a3fffa0d..b491e0ddfb 100644 --- a/_docs/latest/data-studio/transforms/query-transforms.md +++ b/_docs/latest/data-studio/transforms/query-transforms.md @@ -103,7 +103,7 @@ See [Run a transform](transforms-overview#run-a-transform). You'll see logs for ## Incremental query transforms -By default, on every transform run after the first one, Metabase will process all the data in all input tables, then drop the existing target table, and create a new table with the processed data. You can tell Metabase to only write **new** data to your target table by marking your transform as incremental. +By default, on every transform run after the first one, Metabase will process all the data in all input tables, then drop the existing target table, and create a new table with the processed data. You can tell Metabase to only process **new** data by marking your transform as incremental. ### Prerequisites for incremental transforms @@ -113,6 +113,8 @@ Your data has to have certain structure for incremental transforms to work. See For a transform to run incrementally, you'll need to pick a column ("checkpoint") that Metabase needs to check for new values. Then, behind the scenes, Metabase will add a filter around your transform query that will filter the results of the query for values greater than the last written checkpoint value. +If you set a [merge key](transforms-overview#add-merge-keys-to-upsert-rows), Metabase updates the target rows that match the key instead of adding new ones. + ### Make a query transform incremental #### Use table variables for incremental SQL transforms @@ -163,7 +165,8 @@ To make a query transform incremental: 1. Go to the transform's page in **Data studio > Transforms**. 2. Switch to **Settings** tab. -3. In **Column to check for new values**, select the column in one of the source tables that Metabase should check to determine which values are new. Only some columns are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +3. In **Field to check for new values**, select the field in one of the source tables that Metabase should scan to determine which records are new or changed. Only some fields are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +4. (Optional) To update matching rows instead of appending new ones, [add a merge key](./transforms-overview#add-merge-keys-to-upsert-rows). ## Convert models to transforms diff --git a/_docs/latest/data-studio/transforms/transforms-overview.md b/_docs/latest/data-studio/transforms/transforms-overview.md index 79bc5f8faa..82c88f4f1d 100644 --- a/_docs/latest/data-studio/transforms/transforms-overview.md +++ b/_docs/latest/data-studio/transforms/transforms-overview.md @@ -231,13 +231,17 @@ Metabase will track transform dependencies and execute transforms in a reasonabl On Metabase Pro or Enterprise plans, you can see the transform dependencies graph by going to **Data Studio > Transforms** and go to the **Dependencies** tab. -If a job includes a transform that depends on a table created by another transform, then the job will run all the tagged transforms and their dependencies, even if they lack tags, see [Jobs and runs](jobs-and-runs) for more information. +If a job includes a transform that depends on a table created by another transform, then the job will run all the tagged transforms, plus any of their [dependencies that aren't already up to date](jobs-and-runs#jobs-include-all-dependent-transforms). ## Incremental transforms _Data Studio > Transforms > Settings_ -Incremental transforms only append new data since the previous transform run. For example, you might have new transaction data coming in every day, and run the transform nightly. With each run, the incremental transform would only write the rows added after the previous run the night before. +Incremental transforms only process the data that's new since the previous transform run. For example, you might have new transaction data coming in every day, and run the transform nightly. With each run, the incremental transform would only handle the rows added after the previous run the night before. + +By default, Metabase appends those rows to the target table. If your source tables track changes to a record over time, you can use an incremental strategy by setting a [merge key](#add-merge-keys-to-upsert-rows) so that Metabase updates the existing rows instead of adding duplicate rows. + +In this case, the checkpoint field decides which rows count as new, and the merge key decides whether Metabase appends those rows or updates the matching ones. ### Prerequisites for incremental transforms @@ -245,6 +249,30 @@ Incremental transforms only append new data since the previous transform run. Fo - The checkpoint column has to have increasing values, like a sequential ID or timestamp column. Metabase will determine what "new" data is by looking for values that are _greater than_ already-written checkpoint values. - Your schema is stable, meaning that the structure of the tables is not going to change from run to run. +### Add merge keys to upsert rows + +By default, Metabase will append rows when outputting a transform. You can add a merge key to upsert (update or append) instead. + +Why you'd want to add a merge key: some source tables get a new row each time a record changes. So for example the same order shows up once as `created`, again as `paid`, and again as `shipped`. If you append those rows, you end up with three rows for the same order. That may be what you want. But if instead you want to update these records in place, you can select one or more columns as merge keys, like an order ID column. If a merge key is set, Metabase will first try to update existing records that match the key. If no match is found, Metabase will append a new record. + +To add a merge key: + +1. Go to the transform's page in **Data studio > Transforms**. +2. Switch to the **Settings** tab and turn on **Only process new data**. +3. In **Merge key**, choose the columns that identify a record. If the target table already exists, you'll pick from a list of its columns. If the transform hasn't run yet, there's no table to pick from, so type each column name and press comma or enter. In that case, type the column's name in the database, which may differ from the display name you'd see in the list. + +The merge key refers to columns in the _target_ table (the columns your transform outputs), not columns in the _source_ tables. If identifying a record requires a combination of columns, like an `id` plus a `region`, pick more than one column as merge keys. + +Merge keys need a checkpoint field that increases every time a row is written, like an `updated_at` timestamp, otherwise the transform never sees the change. + +### Reprocess all data to rebuild the target table + +Once an incremental transform has run, its **Settings** tab shows **Last processed [checkpoint field]**, followed by the highest checkpoint value Metabase has written so far. That value is how Metabase knows where to pick up, since each run only handles rows past it. + +To clear that value and reprocess all rows, click **Reprocess all data**. The next run will then rebuild the target table from scratch, processing every row instead of only the new ones. The run doesn't start right away; you'll need to run the transform manually, or wait for the transform's next scheduled run. + +Changing the checkpoint field also resets the stored value, so a transform will rebuild from scratch on its next run after you pick a different field to use for checkpointing. + ### Make a transform incremental Incremental transforms work differently for query-based transforms and Python transforms, so see [incremental query transforms](query-transforms#incremental-query-transforms) and [incremental Python transforms](./python-transforms#incremental-python-transforms) for more information. diff --git a/_docs/v0.63/configuring-metabase/config-template.md b/_docs/v0.63/configuring-metabase/config-template.md index e413e54a9c..410ea413bd 100644 --- a/_docs/v0.63/configuring-metabase/config-template.md +++ b/_docs/v0.63/configuring-metabase/config-template.md @@ -217,8 +217,26 @@ config: ldap-user-filter: (&(objectClass=inetOrgPerson)(|(uid={login})(mail={login}))) ldap-user-provisioning-enabled: true license-token-missing-banner-dismissal-timestamp: [] + llm-anthropic-api-base-url: https://api.anthropic.com llm-anthropic-api-key: null + llm-anthropic-model: claude-opus-4-5-20251101 + llm-azure-api-base-url: null + llm-azure-api-key: null + llm-bedrock-access-key-id: null + llm-bedrock-region: us-east-1 + llm-bedrock-secret-access-key: null + llm-bedrock-session-token: null + llm-connection-timeout-ms: 5000 + llm-max-tokens: 4096 llm-metabot-provider: anthropic/claude-sonnet-4-6 + llm-openai-api-base-url: https://api.openai.com + llm-openai-api-key: null + llm-openai-model: gpt-5.4 + llm-openrouter-api-base-url: https://openrouter.ai/api + llm-openrouter-api-key: null + llm-rate-limit-per-ip: 100 + llm-rate-limit-per-user: 20 + llm-request-timeout-ms: 60000 load-analytics-content: true loading-message: doing-science login-page-illustration: default diff --git a/_docs/v0.63/configuring-metabase/environment-variables.md b/_docs/v0.63/configuring-metabase/environment-variables.md index 49789acc44..99e52a2928 100644 --- a/_docs/v0.63/configuring-metabase/environment-variables.md +++ b/_docs/v0.63/configuring-metabase/environment-variables.md @@ -1294,6 +1294,14 @@ When set to `true`, users who log in via LDAP will automatically get a Metabase The array of last two ISO8601 dates when an admin dismissed the license token missing banner. +### `MB_LLM_ANTHROPIC_API_BASE_URL` + +- Type: string +- Default: `https://api.anthropic.com` +- [Configuration file name](./config-file): `llm-anthropic-api-base-url` + +The Anthropic API base URL. + ### `MB_LLM_ANTHROPIC_API_KEY` - Type: string @@ -1302,6 +1310,78 @@ The array of last two ISO8601 dates when an admin dismissed the license token mi The Anthropic API Key. +### `MB_LLM_ANTHROPIC_MODEL` + +- Type: string +- Default: `claude-opus-4-5-20251101` +- [Configuration file name](./config-file): `llm-anthropic-model` + +The Anthropic model to use. + +### `MB_LLM_AZURE_API_BASE_URL` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-azure-api-base-url` + +The base URL of the Azure resource's OpenAI- or Anthropic-compatible surface, e.g. `https://.services.ai.azure.com/openai`. + +### `MB_LLM_AZURE_API_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-azure-api-key` + +The API key for the Azure resource hosting your models. + +### `MB_LLM_BEDROCK_ACCESS_KEY_ID` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-bedrock-access-key-id` + +The AWS Access Key ID for Amazon Bedrock. + +### `MB_LLM_BEDROCK_REGION` + +- Type: string +- Default: `us-east-1` +- [Configuration file name](./config-file): `llm-bedrock-region` + +The AWS region for Amazon Bedrock (e.g. us-east-1). + +### `MB_LLM_BEDROCK_SECRET_ACCESS_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-bedrock-secret-access-key` + +The AWS Secret Access Key for Amazon Bedrock. + +### `MB_LLM_BEDROCK_SESSION_TOKEN` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-bedrock-session-token` + +The AWS Session Token for Amazon Bedrock. Only needed for temporary credentials. + +### `MB_LLM_CONNECTION_TIMEOUT_MS` + +- Type: integer +- Default: `5000` +- [Configuration file name](./config-file): `llm-connection-timeout-ms` + +Connection timeout in milliseconds for LLM API requests. + +### `MB_LLM_MAX_TOKENS` + +- Type: integer +- Default: `4096` +- [Configuration file name](./config-file): `llm-max-tokens` + +Maximum tokens for LLM responses. + ### `MB_LLM_METABOT_PROVIDER` - Type: string @@ -1310,6 +1390,70 @@ The Anthropic API Key. The AI provider and model for Metabot. Format: provider/model-name, e.g. `anthropic/claude-haiku-4-5`, `openai/gpt-5.4`, `openrouter/anthropic/claude-haiku-4.5`. +### `MB_LLM_OPENAI_API_BASE_URL` + +- Type: string +- Default: `https://api.openai.com` +- [Configuration file name](./config-file): `llm-openai-api-base-url` + +The OpenAI API base URL. + +### `MB_LLM_OPENAI_API_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-openai-api-key` + +The OpenAI API Key. + +### `MB_LLM_OPENAI_MODEL` + +- Type: string +- Default: `gpt-5.4` +- [Configuration file name](./config-file): `llm-openai-model` + +The OpenAI Model (e.g. 'gpt-5.5', 'gpt-5.4-mini'). + +### `MB_LLM_OPENROUTER_API_BASE_URL` + +- Type: string +- Default: `https://openrouter.ai/api` +- [Configuration file name](./config-file): `llm-openrouter-api-base-url` + +The OpenRouter API base URL used for Chat Completions. + +### `MB_LLM_OPENROUTER_API_KEY` + +- Type: string +- Default: `null` +- [Configuration file name](./config-file): `llm-openrouter-api-key` + +The OpenRouter API Key. + +### `MB_LLM_RATE_LIMIT_PER_IP` + +- Type: integer +- Default: `100` +- [Configuration file name](./config-file): `llm-rate-limit-per-ip` + +Maximum SQL generation requests per IP address per minute. + +### `MB_LLM_RATE_LIMIT_PER_USER` + +- Type: integer +- Default: `20` +- [Configuration file name](./config-file): `llm-rate-limit-per-user` + +Maximum SQL generation requests per user per minute. + +### `MB_LLM_REQUEST_TIMEOUT_MS` + +- Type: integer +- Default: `60000` +- [Configuration file name](./config-file): `llm-request-timeout-ms` + +Socket timeout in milliseconds for LLM API requests. + ### `MB_LOAD_ANALYTICS_CONTENT` - Type: boolean diff --git a/_docs/v0.63/data-studio/transforms/jobs-and-runs.md b/_docs/v0.63/data-studio/transforms/jobs-and-runs.md index d6264fcfff..0d920f8abc 100644 --- a/_docs/v0.63/data-studio/transforms/jobs-and-runs.md +++ b/_docs/v0.63/data-studio/transforms/jobs-and-runs.md @@ -45,7 +45,7 @@ Jobs have two components: schedule and tags. - **Schedule** determines when the job will be executed: daily, hourly, etc. You can specify a custom cron schedule (e.g. "Every weekday at 9:05 AM"). The times are given in your Metabase's system timezone. - **Tags** determine _which_ transforms a job runs, not when the job runs. For example, you can create a `Weekdays` tag, add that tag to a few transforms, then create a job that runs all the transforms with the `Weekdays` tag every weekday at 9:05AM. -Jobs will run all transforms tagged with any of the tags, plus any transforms that the tagged transforms depend on, see [Jobs will run all dependent transforms](#jobs-will-run-all-dependent-transforms). +Jobs will run all transforms tagged with any of the tags, plus any transforms that the tagged transforms depend on that aren't already up to date, see [Jobs include all dependent transforms](#jobs-include-all-dependent-transforms). You can see which transforms a job will run and in which order on the job's page. @@ -111,13 +111,22 @@ To delete a job: 2. Find the job you want to delete and click the **three dots** icon to the right of the job's name. 3. Select **Delete**. -## Jobs will run all dependent transforms +## Jobs include all dependent transforms -If one transform depends on another, Metabase will run the dependency first, even if that transform isn't tagged in the job. So if transform B depends on A, Metabase will first run A, even if A doesn't have a tag. +If one transform depends on another, Metabase adds the dependency to the job and runs it first, even if that transform isn't tagged in the job. So if transform B depends on A, Metabase makes sure A is up to date before running B. -This means that you can explicitly tag transform A to run daily, and transform B hourly, but because transform B depends on transform A, transform A will _also_ run hourly (in addition to daily), despite not having the tag. +## Jobs skip dependencies that are already up to date -You can see which transforms a job will run (and in which order) on the job's page in **Data Studio > Jobs**. +Metabase won't re-run a dependency that's still fresh. That way, a transform pulled into a frequent job doesn't get rebuilt more often than its own schedule calls for. So, for example, if you tag transform A `daily`, and transform B `hourly`, and hourly B depends on daily A, then A runs on its own daily schedule. The hourly job that runs B won't rerun A every hour. + +Metabase skips a dependency when the dependency: + +- Has its own tags and jobs, and none of those jobs' schedules have come due since the dependency last ran successfully. +- Has no tags at all, and it's already run successfully at least once. + +## See which transforms a job will run + +The job's page in **Data Studio > Jobs** lists every transform the job will run and in which order. The **Notes** column tells you when Metabase will skip a dependency, and flags dependencies that have no schedule of their own. ## Runs @@ -127,4 +136,11 @@ You can see all past and current transform runs (both manual and scheduled) by g You can click on any transform run to see more details about the run, like the error logs. To go to the transform definition from the transform run page, click on the icon next the transform name in the right sidebar. -The "Tags" column in the **Runs** table will only show the transform's specific tags. But the run might not have anything to do with those tags. Another job with different tags could have run the transform because [jobs will run _all_ dependent transforms](#jobs-will-run-all-dependent-transforms). +The "Tags" column in the **Runs** table will only show the transform's specific tags. But the run might not have anything to do with those tags. Another job with different tags could have run the transform because [jobs include all dependent transforms](#jobs-include-all-dependent-transforms). + +## Emails about failed transforms + +If your Metabase is [set up to send email](../../configuring-metabase/email), Metabase will let people know when transforms fail. + +- **Individual transform failures**: when transforms fail during a scheduled job run, Metabase emails the last person to edit each failed transform (or the transform's creator, or admins if neither is active). Each email covers a single job run. +- **Daily digest of job failures**: each morning, Metabase emails all admins a summary of the scheduled job runs that failed or timed out the previous day. Manual runs aren't included in the digest, and Metabase skips the email entirely if no scheduled runs failed. diff --git a/_docs/v0.63/data-studio/transforms/python-transforms.md b/_docs/v0.63/data-studio/transforms/python-transforms.md index 05ca613370..92c480a654 100644 --- a/_docs/v0.63/data-studio/transforms/python-transforms.md +++ b/_docs/v0.63/data-studio/transforms/python-transforms.md @@ -115,7 +115,7 @@ To use functions or classes from your Python library: ## Incremental Python transforms -By default, Metabase will process all the data in all input tables, drop the existing target table (if one exists), and create a new table with the processed data. You can tell Metabase to only write **new** data to your target table by marking your transform as incremental. +By default, Metabase will process all the data in all input tables, drop the existing target table (if one exists), and create a new table with the processed data. You can tell Metabase to only process **new** data by marking your transform as incremental. ### Prerequisites for incremental transforms @@ -127,7 +127,8 @@ To make a Python transform incremental: 1. Go to the transform's page in **Data studio > Transforms**. 2. Switch to **Settings** tab. -3. In **Column to check for new values**, select the column in one of the source tables that Metabase should check to determine which values are new. Only some columns are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +3. In **Field to check for new values**, select the field in one of the source tables that Metabase should scan to determine which records are new or changed. Only some fields are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +4. (Optional) To update matching rows instead of appending new ones, [add a merge key](./transforms-overview#add-merge-keys-to-upsert-rows). ## Current limitations of Python transforms diff --git a/_docs/v0.63/data-studio/transforms/query-transforms.md b/_docs/v0.63/data-studio/transforms/query-transforms.md index 74a5a6da4f..b62748e0a0 100644 --- a/_docs/v0.63/data-studio/transforms/query-transforms.md +++ b/_docs/v0.63/data-studio/transforms/query-transforms.md @@ -99,7 +99,7 @@ See [Run a transform](transforms-overview#run-a-transform). You'll see logs for ## Incremental query transforms -By default, on every transform run after the first one, Metabase will process all the data in all input tables, then drop the existing target table, and create a new table with the processed data. You can tell Metabase to only write **new** data to your target table by marking your transform as incremental. +By default, on every transform run after the first one, Metabase will process all the data in all input tables, then drop the existing target table, and create a new table with the processed data. You can tell Metabase to only process **new** data by marking your transform as incremental. ### Prerequisites for incremental transforms @@ -109,6 +109,8 @@ Your data has to have certain structure for incremental transforms to work. See For a transform to run incrementally, you'll need to pick a column ("checkpoint") that Metabase needs to check for new values. Then, behind the scenes, Metabase will add a filter around your transform query that will filter the results of the query for values greater than the last written checkpoint value. +If you set a [merge key](transforms-overview#add-merge-keys-to-upsert-rows), Metabase updates the target rows that match the key instead of adding new ones. + ### Make a query transform incremental #### Use table variables for incremental SQL transforms @@ -159,7 +161,8 @@ To make a query transform incremental: 1. Go to the transform's page in **Data studio > Transforms**. 2. Switch to **Settings** tab. -3. In **Column to check for new values**, select the column in one of the source tables that Metabase should check to determine which values are new. Only some columns are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +3. In **Field to check for new values**, select the field in one of the source tables that Metabase should scan to determine which records are new or changed. Only some fields are eligible. See [prerequisites for incremental transforms](./transforms-overview#prerequisites-for-incremental-transforms). +4. (Optional) To update matching rows instead of appending new ones, [add a merge key](./transforms-overview#add-merge-keys-to-upsert-rows). ## Convert models to transforms diff --git a/_docs/v0.63/data-studio/transforms/transforms-overview.md b/_docs/v0.63/data-studio/transforms/transforms-overview.md index 7c9c71fa11..b3755ed300 100644 --- a/_docs/v0.63/data-studio/transforms/transforms-overview.md +++ b/_docs/v0.63/data-studio/transforms/transforms-overview.md @@ -227,13 +227,17 @@ Metabase will track transform dependencies and execute transforms in a reasonabl On Metabase Pro or Enterprise plans, you can see the transform dependencies graph by going to **Data Studio > Transforms** and go to the **Dependencies** tab. -If a job includes a transform that depends on a table created by another transform, then the job will run all the tagged transforms and their dependencies, even if they lack tags, see [Jobs and runs](jobs-and-runs) for more information. +If a job includes a transform that depends on a table created by another transform, then the job will run all the tagged transforms, plus any of their [dependencies that aren't already up to date](jobs-and-runs#jobs-include-all-dependent-transforms). ## Incremental transforms _Data Studio > Transforms > Settings_ -Incremental transforms only append new data since the previous transform run. For example, you might have new transaction data coming in every day, and run the transform nightly. With each run, the incremental transform would only write the rows added after the previous run the night before. +Incremental transforms only process the data that's new since the previous transform run. For example, you might have new transaction data coming in every day, and run the transform nightly. With each run, the incremental transform would only handle the rows added after the previous run the night before. + +By default, Metabase appends those rows to the target table. If your source tables track changes to a record over time, you can use an incremental strategy by setting a [merge key](#add-merge-keys-to-upsert-rows) so that Metabase updates the existing rows instead of adding duplicate rows. + +In this case, the checkpoint field decides which rows count as new, and the merge key decides whether Metabase appends those rows or updates the matching ones. ### Prerequisites for incremental transforms @@ -241,6 +245,30 @@ Incremental transforms only append new data since the previous transform run. Fo - The checkpoint column has to have increasing values, like a sequential ID or timestamp column. Metabase will determine what "new" data is by looking for values that are _greater than_ already-written checkpoint values. - Your schema is stable, meaning that the structure of the tables is not going to change from run to run. +### Add merge keys to upsert rows + +By default, Metabase will append rows when outputting a transform. You can add a merge key to upsert (update or append) instead. + +Why you'd want to add a merge key: some source tables get a new row each time a record changes. So for example the same order shows up once as `created`, again as `paid`, and again as `shipped`. If you append those rows, you end up with three rows for the same order. That may be what you want. But if instead you want to update these records in place, you can select one or more columns as merge keys, like an order ID column. If a merge key is set, Metabase will first try to update existing records that match the key. If no match is found, Metabase will append a new record. + +To add a merge key: + +1. Go to the transform's page in **Data studio > Transforms**. +2. Switch to the **Settings** tab and turn on **Only process new data**. +3. In **Merge key**, choose the columns that identify a record. If the target table already exists, you'll pick from a list of its columns. If the transform hasn't run yet, there's no table to pick from, so type each column name and press comma or enter. In that case, type the column's name in the database, which may differ from the display name you'd see in the list. + +The merge key refers to columns in the _target_ table (the columns your transform outputs), not columns in the _source_ tables. If identifying a record requires a combination of columns, like an `id` plus a `region`, pick more than one column as merge keys. + +Merge keys need a checkpoint field that increases every time a row is written, like an `updated_at` timestamp, otherwise the transform never sees the change. + +### Reprocess all data to rebuild the target table + +Once an incremental transform has run, its **Settings** tab shows **Last processed [checkpoint field]**, followed by the highest checkpoint value Metabase has written so far. That value is how Metabase knows where to pick up, since each run only handles rows past it. + +To clear that value and reprocess all rows, click **Reprocess all data**. The next run will then rebuild the target table from scratch, processing every row instead of only the new ones. The run doesn't start right away; you'll need to run the transform manually, or wait for the transform's next scheduled run. + +Changing the checkpoint field also resets the stored value, so a transform will rebuild from scratch on its next run after you pick a different field to use for checkpointing. + ### Make a transform incremental Incremental transforms work differently for query-based transforms and Python transforms, so see [incremental query transforms](query-transforms#incremental-query-transforms) and [incremental Python transforms](./python-transforms#incremental-python-transforms) for more information.