From 342aa2157675d23b17cb163d88d833710451eec6 Mon Sep 17 00:00:00 2001 From: mehulsonowal Date: Fri, 9 Oct 2026 12:50:46 -0400 Subject: [PATCH 1/3] [DOCS-15714] Update Agent Observability experiments guide --- hugo/config/_default/menus/main.en.yaml | 25 - .../en/llm_observability/guide/_index.md | 4 + .../advanced_experiment_runs.md} | 3 +- .../analyzing_experiment_results.md} | 1 + .../en/llm_observability/guide/experiments.md | 742 ++++++++++++++++++ .../api.md => guide/experiments_api.md} | 3 +- .../prompt_optimization.md | 3 +- .../improve/experiments/_index.md | 2 +- .../improve/experiments/setup.md | 493 ------------ .../llm_observability/improve/playground.md | 2 +- .../instrument/otel_instrumentation.md | 2 +- .../external_evaluations/deepeval.md | 4 +- .../external_evaluations/pydantic.md | 4 +- 13 files changed, 760 insertions(+), 528 deletions(-) rename hugo/content/en/llm_observability/{improve/experiments/advanced_runs.md => guide/advanced_experiment_runs.md} (99%) rename hugo/content/en/llm_observability/{improve/experiments/analyzing_results.md => guide/analyzing_experiment_results.md} (99%) create mode 100644 hugo/content/en/llm_observability/guide/experiments.md rename hugo/content/en/llm_observability/{improve/experiments/api.md => guide/experiments_api.md} (99%) rename hugo/content/en/llm_observability/{improve/experiments => guide}/prompt_optimization.md (99%) delete mode 100644 hugo/content/en/llm_observability/improve/experiments/setup.md diff --git a/hugo/config/_default/menus/main.en.yaml b/hugo/config/_default/menus/main.en.yaml index b6e1e544c4d..0be797e886b 100644 --- a/hugo/config/_default/menus/main.en.yaml +++ b/hugo/config/_default/menus/main.en.yaml @@ -5868,31 +5868,6 @@ menu: parent: llm_obs_improve identifier: llm_obs_experiments weight: 503 - - name: Setup and Usage - url: llm_observability/improve/experiments/setup - parent: llm_obs_experiments - identifier: llm_obs_experiments_setup - weight: 50301 - - name: Analyzing Results - url: llm_observability/improve/experiments/analyzing_results - parent: llm_obs_experiments - identifier: llm_obs_experiments_analyzing_results - weight: 50302 - - name: Advanced Experiment Runs - url: llm_observability/improve/experiments/advanced_runs - parent: llm_obs_experiments - identifier: llm_obs_experiments_advanced_runs - weight: 50303 - - name: Prompt Optimization - url: llm_observability/improve/experiments/prompt_optimization - parent: llm_obs_experiments - identifier: llm_obs_experiments_prompt_optimization - weight: 50304 - - name: Experiments API - url: llm_observability/improve/experiments/api - parent: llm_obs_experiments - identifier: llm_obs_experiments_api - weight: 50305 - name: Data Access Control url: llm_observability/improve/access_control parent: llm_obs_improve diff --git a/hugo/content/en/llm_observability/guide/_index.md b/hugo/content/en/llm_observability/guide/_index.md index 0987cb5b77b..0871d0c2a34 100644 --- a/hugo/content/en/llm_observability/guide/_index.md +++ b/hugo/content/en/llm_observability/guide/_index.md @@ -12,6 +12,10 @@ cascade: {{< nextlink href="/llm_observability/quickstart/" >}}Trace an LLM Application{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/trace-an-llm-application-in-aws-lambda" >}}Trace an LLM Application in AWS Lambda{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/monitor_proxy_services" >}}Trace Proxy and Gateway Services{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/experiments" >}}Set up and use Agent Observability Experiments{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/advanced_experiment_runs" >}}Advanced experiment runs{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/analyzing_experiment_results" >}}Analyze experiment results{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/experiments_api" >}}Experiments API{{< /nextlink >}} {{< nextlink href="/llm_observability/instrument/agent_observability_and_apm" >}}Using Agent Observability and APM{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/monitor_mcp_client" >}}Monitor MCP Clients{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/crewai_guide" >}}Using the Datadog-CrewAI integration for Agent Observability{{< /nextlink >}} diff --git a/hugo/content/en/llm_observability/improve/experiments/advanced_runs.md b/hugo/content/en/llm_observability/guide/advanced_experiment_runs.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/advanced_runs.md rename to hugo/content/en/llm_observability/guide/advanced_experiment_runs.md index 5aa92e7207d..8707e6b5d68 100644 --- a/hugo/content/en/llm_observability/improve/experiments/advanced_runs.md +++ b/hugo/content/en/llm_observability/guide/advanced_experiment_runs.md @@ -1,6 +1,7 @@ --- title: Advanced Experiment Runs aliases: +- /llm_observability/improve/experiments/advanced_runs/ - /llm_observability/experiments/advanced_runs/ description: Run experiments multiple times to account for model variability on a subset of your dataset, and automate experiment execution in CI/CD pipelines. --- @@ -271,5 +272,5 @@ GET /api/v2/llm-obs/v1/experiments?filter[experiment]=my-pipeline&filter[metadat ``` [1]: /llm_observability/instrument/sdk?tab=python -[2]: /llm_observability/improve/experiments/api +[2]: /llm_observability/guide/experiments_api [3]: https://app.datadoghq.com/llm/experiments diff --git a/hugo/content/en/llm_observability/improve/experiments/analyzing_results.md b/hugo/content/en/llm_observability/guide/analyzing_experiment_results.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/analyzing_results.md rename to hugo/content/en/llm_observability/guide/analyzing_experiment_results.md index 3fe75ce9d25..9082f6ec23f 100644 --- a/hugo/content/en/llm_observability/improve/experiments/analyzing_results.md +++ b/hugo/content/en/llm_observability/guide/analyzing_experiment_results.md @@ -1,6 +1,7 @@ --- title: Analyze Your Experiments Results aliases: +- /llm_observability/improve/experiments/analyzing_results/ - /llm_observability/experiments/analyzing_results/ description: How to analyze Agent Observability Experiments results. --- diff --git a/hugo/content/en/llm_observability/guide/experiments.md b/hugo/content/en/llm_observability/guide/experiments.md new file mode 100644 index 00000000000..ab2516dccb3 --- /dev/null +++ b/hugo/content/en/llm_observability/guide/experiments.md @@ -0,0 +1,742 @@ +--- +title: Set Up and Use Agent Observability Experiments +aliases: +- /llm_observability/improve/experiments/setup/ +- /llm_observability/experiments/setup/ +description: How to set up Agent Observability Experiments and start running experiments. +further_reading: +- link: "https://www.datadoghq.com/blog/debug-and-evaluate-your-ai-app-from-your-coding-agent/" + tag: "Blog" + text: "Debug and evaluate your AI app from your coding agent with Datadog Agent Observability" +--- + +This guide describes how to set up and use Agent Observability experiments with the Python or Node.js SDK. For complete runnable Node.js examples, see the [Node.js experiments examples](https://github.com/DataDog/llm-observability/tree/main/experiments/nodejs). + +## Prerequisites + +To run experiments, you need a Datadog API key and application key. + +## Set up Agent Observability + +If you have not already set up Agent Observability, select the Python or Node.js tab and follow the instructions for your application's language. + +Replace `site` in the following examples with your Datadog site: {{< region-param key="dd_site" code="true" >}}. + +1. Install the Agent Observability SDK: + + {{< tabs >}} +{{% tab "Python" %}} +```shell +pip install ddtrace>=4.3.0 +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```shell +npm install "dd-trace@>=6.13.0" +``` +{{% /tab %}} + {{< /tabs >}} + +2. Enable Agent Observability: + + {{< tabs >}} +{{% tab "Python" %}} +Pass `api_key` and `app_key` to `LLMObs.enable()`, or set the `DD_API_KEY` and `DD_APP_KEY` environment variables: + +```python +from ddtrace.llmobs import LLMObs + +LLMObs.enable( + api_key="", # defaults to DD_API_KEY environment variable + app_key="", # defaults to DD_APP_KEY environment variable + site="datadoghq.com", # defaults to DD_SITE environment variable + project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set +) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +Set `DD_API_KEY` and `DD_APP_KEY` in your environment. The Experiments client uses these variables for authentication. Then choose one of the following initialization methods. + +**In-code initialization** + +Initialize `dd-trace` in your application entrypoint: + +```javascript +const tracer = require('dd-trace').init({ + service: '', + site: 'datadoghq.com', + llmobs: { + mlApp: '', + projectName: '', + agentlessEnabled: true, + }, +}) + +const { experiments } = tracer.llmobs +``` + +Configure the project and application names independently: + +- `projectName` identifies the project that contains your datasets and experiments. If omitted, it defaults to `default-project`. +- `mlApp` identifies the LLM application used for Agent Observability traces. Neither `mlApp` nor `service` determines the Experiments project name. + +**Command-line initialization** + +Alternatively, initialize `dd-trace` from the command line instead of calling `require('dd-trace').init(...)` in your application. Set the configuration and authentication environment variables when starting your application: + +```shell +DD_SITE= DD_API_KEY= DD_APP_KEY= \ +DD_LLMOBS_ENABLED=1 DD_LLMOBS_ML_APP= DD_LLMOBS_PROJECT_NAME= \ +NODE_OPTIONS="--import dd-trace/initialize.mjs" node +``` + +For more information, see the [Node.js tracer command-line setup](/llm_observability/instrument/sdk?tab=nodejs#command-line-setup). +{{% /tab %}} + {{< /tabs >}} + +### APM Trace correlation + +To correlate experiment spans with [APM traces](/llm_observability/instrument/agent_observability_and_apm/), run Agent Observability through a Datadog Agent. The Agent forwards trace data to APM. + +{{< tabs >}} +{{% tab "Python" %}} +APM trace correlation requires `agentless_enabled=False`, which is the default: + +```python +LLMObs.enable( + api_key="", + app_key="", + site="datadoghq.com", + agentless_enabled=False, # default — required for APM trace correlation + project_name="", +) +``` + +To run without a Datadog Agent, set `agentless_enabled` to `True`. Agentless runs do not generate corresponding APM spans for experiment spans. +{{% /tab %}} + +{{% tab "Node.js" %}} +APM trace correlation requires `agentlessEnabled: false`, which is the default. If you used the in-code initialization example in the setup section, change `agentlessEnabled` from `true` to `false`: + +```javascript +const tracer = require('dd-trace').init({ + service: '', + site: 'datadoghq.com', + llmobs: { + mlApp: '', + projectName: '', + agentlessEnabled: false, // default — required for APM trace correlation + }, +}) +``` + +To run without a Datadog Agent, set `agentlessEnabled` to `true`. Agentless runs do not generate corresponding APM spans for experiment spans. +{{% /tab %}} +{{< /tabs >}} + +## Create a project + +Projects organize your datasets and experiments. You can create a project in the Datadog UI or through the API. To create a project with an SDK, specify a project name that does not already exist. + +{{< tabs >}} +{{% tab "Python" %}} +```python +from ddtrace.llmobs import LLMObs + +LLMObs.enable( + ..., + project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set +) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const tracer = require('dd-trace').init({ + llmobs: { + mlApp: '', + projectName: 'experiments-project', + }, +}) + +const { experiments } = tracer.llmobs +``` + +Set the default project with `llmobs.projectName` or the `DD_LLMOBS_PROJECT_NAME` environment variable. The Experiments client creates the project when it first accesses it. + +To use a different project for a specific operation, pass `projectName` to that operation. For example, pass it to `pullDataset()` to retrieve a dataset from another project. When creating a dataset and its experiment, use the same project name for both: + +```javascript +async function main () { + const dataset = experiments.createDataset('capital-cities', { + projectName: 'other-project', + records: [ + { inputData: { country: 'France' }, expectedOutput: 'Paris' }, + { inputData: { country: 'Japan' }, expectedOutput: 'Tokyo' }, + ], + }) + + const experiment = experiments.experiment({ + name: 'capital-cities-test', + projectName: 'other-project', + dataset, + task: (inputData) => ({ France: 'Paris', Japan: 'Tokyo' })[inputData.country], + evaluators: { + exact_match: (_inputData, outputData, expectedOutput) => outputData === expectedOutput, + }, + }) + + const result = await experiment.run() + console.log(`View experiment: ${result.url}`) +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) +``` +{{% /tab %}} +{{< /tabs >}} + +## Create a dataset + +A dataset is a collection of inputs, expected outputs, and metadata that represent scenarios you want to test your agent on. Each dataset belongs to a project. + +- **Input** (required): Represents all the information that the agent can access in a task. +- **Expected output** (optional): Also called _ground truth_, represents the ideal answer that the agent should output. You can use _expected output_ to store the actual output of the app, as well as any intermediary results you want to assess. +- **Metadata** (optional): Contains information to categorize the record for further analysis, such as topics, tags, descriptions, and notes. + +Create a dataset using the example for your SDK: + +{{< tabs >}} +{{% tab "Python" %}} +To create a dataset from a CSV file, use `LLMObs.create_dataset_from_csv()`: + +```python +# Create dataset from CSV +dataset = LLMObs.create_dataset_from_csv( + csv_path="questions.csv", + dataset_name="capitals-of-the-world", + project_name="capitals-project", # Optional: defaults to the project name from LLMObs.enable + description="Geography quiz dataset", # Optional: Dataset description + input_data_columns=["question", "category"], # Columns to use as input + expected_output_columns=["answer"], # Optional: Columns to use as expected output + metadata_columns=["difficulty"], # Optional: Additional columns as metadata + csv_delimiter="," # Optional: Defaults to comma +) + +# Example "questions.csv": +# question,category,answer,difficulty +# What is the capital of Japan?,geography,Tokyo,medium +# What is the capital of Brazil?,geography,Brasília,medium +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +The Node.js SDK does not include a CSV-specific helper. Parse the CSV with the library of your choice, then pass the records to `createDataset()`: + +```javascript +const dataset = experiments.createDataset('capitals-of-the-world', { + description: 'Geography quiz dataset', + records: [ + { + inputData: { question: 'What is the capital of Japan?', category: 'geography' }, + expectedOutput: 'Tokyo', + metadata: { difficulty: 'medium' }, + }, + { + inputData: { question: 'What is the capital of Brazil?', category: 'geography' }, + expectedOutput: 'Brasília', + metadata: { difficulty: 'medium' }, + }, + ], +}) + +// The dataset is pushed automatically when the experiment runs. +// To push it earlier, use: await dataset.push() +``` +{{% /tab %}} +{{< /tabs >}} + +See [Datasets][1] for more information about datasets, including how to manually create datasets, retrieve and manage datasets, and how Datadog retains dataset versions. + +## Create an experiment + +An experiment runs your agent across scenarios in a dataset and uses evaluators to measure its performance. Compare experiments to assess how different application configurations perform. + +An experiment has the following components: + +- **Task**: Defines the core workflow you want to evaluate. It can range from a single LLM call to a more complex flow involving multiple LLM calls and RAG steps. The task is executed sequentially across all records in the dataset. +- **Evaluator**: A function, executed on each record, that measures how well the model or agent performs. Evaluators allow you to compare the output to either the expected output or the original input. +- **Summary evaluators**: Optional functions executed against all the data of the experiment (input, output, expected output, and evaluator results). Summary evaluators allow you to compute more advanced metrics like precision, recall, and accuracy across your dataset. + +To create an experiment: + +### 1. Load a dataset + +{{< tabs >}} +{{% tab "Python" %}} +```python +from ddtrace.llmobs import LLMObs +from typing import Dict, Any, Optional, List + +dataset = LLMObs.pull_dataset("capitals-of-the-world") +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const tracer = require('dd-trace').init() +const { experiments } = tracer.llmobs + +const dataset = await experiments.pullDataset('capitals-of-the-world') +``` +{{% /tab %}} +{{< /tabs >}} + +### 2. Define a task function that processes a single dataset record + +{{< tabs >}} +{{% tab "Python" %}} +```python +def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: + question = input_data["question"] + # Your LLM or processing logic here + return "Beijing" if "China" in question else "Unknown" +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +async function task (inputData, config, metadata) { + const question = inputData.question + // Your LLM or processing logic here + return question.includes('China') ? 'Beijing' : 'Unknown' +} +``` +{{% /tab %}} +{{< /tabs >}} + +A task can take any non-null type as `input_data` or `inputData` (string, number, Boolean, object, or array). Evaluators can use output of any type. This example returns a string. To include intermediate results for evaluation, return a dictionary or object. + +Optionally, your task function can accept metadata from the dataset record: + +{{< tabs >}} +{{% tab "Python" %}} +```python +def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None, metadata: Optional[Dict[str, Any]] = None) -> str: + difficulty = metadata.get("difficulty", "unknown") if metadata else "unknown" + question = input_data["question"] + return "Beijing" if "China" in question else "Unknown" +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +function task (inputData, config, metadata) { + const difficulty = metadata?.difficulty ?? 'unknown' + const question = inputData.question + // Use difficulty in the task when it is part of your evaluation logic. + return question.includes('China') ? 'Beijing' : 'Unknown' +} +``` +{{% /tab %}} +{{< /tabs >}} + +Trace parts of your experiment task, for example, workflows or tool calls, using the same tracing APIs you use in production. Agent Observability also automatically traces and annotates calls to supported LLM frameworks and libraries. + +{{< tabs >}} +{{% tab "Python" %}} +For custom tracing, use the [Python tracing decorators][2]. For automatic instrumentation, see the [supported Python frameworks][3]. + +[2]: /llm_observability/instrument/sdk/?tab=python#manual-instrumentation +[3]: /llm_observability/instrument/auto_instrumentation?tab=python + +```python +# Example: trace a workflow or tool call with the same decorators used in production. +from ddtrace.llmobs.decorators import workflow + +@workflow(name="retrieve_context") +def retrieve_context(question): + return call_retriever(question) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +For custom tracing, use `llmobs.trace()`. For automatic instrumentation, see the [supported Node.js frameworks](/llm_observability/instrument/auto_instrumentation?tab=nodejs). + +```javascript +// Example: trace a workflow or tool call with the same LLMObs APIs used in production. +const context = await tracer.llmobs.trace( + { kind: 'task', name: 'retrieve_context' }, + async () => callRetriever(question), +) +``` +{{% /tab %}} +{{< /tabs >}} + +#### Using OpenTelemetry spans inside experiments + +If your application uses [OpenTelemetry instrumentation][6], you can create OTel spans inside your experiment task. With `DD_TRACE_OTEL_ENABLED=1`, the Datadog tracer acts as the OpenTelemetry TracerProvider, so OTel spans appear as children of the experiment span automatically. + +{{< tabs >}} +{{% tab "Python" %}} +```python +import json +from opentelemetry import trace + +tracer = trace.get_tracer(__name__) + +def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: + question = input_data["question"] + + # OTel gen_ai span — automatically becomes a child of the experiment span + with tracer.start_as_current_span("my-llm-call") as span: + span.set_attribute("gen_ai.operation.name", "chat") + span.set_attribute("gen_ai.system", "openai") + span.set_attribute("gen_ai.request.model", "gpt-4o") + span.set_attribute("gen_ai.usage.input_tokens", 25) + span.set_attribute("gen_ai.usage.output_tokens", 8) + span.set_attribute( + "gen_ai.input.messages", + json.dumps([{"role": "user", "parts": [{"type": "text", "content": question}]}]), + ) + + result = call_my_llm(question) + + span.set_attribute( + "gen_ai.output.messages", + json.dumps([{"role": "assistant", "parts": [{"type": "text", "content": result}]}]), + ) + + return result +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +Install the OpenTelemetry API if it is not already a dependency: + +```shell +npm install @opentelemetry/api +``` + +```javascript +const { trace } = require('@opentelemetry/api') + +const otelTracer = trace.getTracer('my-experiment') + +async function task (inputData, config, metadata) { + const question = inputData.question + + return otelTracer.startActiveSpan('my-llm-call', async (span) => { + span.setAttribute('gen_ai.operation.name', 'chat') + span.setAttribute('gen_ai.system', 'openai') + span.setAttribute('gen_ai.request.model', 'gpt-4o') + span.setAttribute('gen_ai.usage.input_tokens', 25) + span.setAttribute('gen_ai.usage.output_tokens', 8) + span.setAttribute('gen_ai.input.messages', JSON.stringify([ + { role: 'user', parts: [{ type: 'text', content: question }] }, + ])) + + const result = await callMyLlm(question) + + span.setAttribute('gen_ai.output.messages', JSON.stringify([ + { role: 'assistant', parts: [{ type: 'text', content: result }] }, + ])) + span.end() + return result + }) +} +``` +{{% /tab %}} +{{< /tabs >}} + +To enable this, set the `DD_TRACE_OTEL_ENABLED` environment variable: + +{{< tabs >}} +{{% tab "Python" %}} +```shell +DD_TRACE_OTEL_ENABLED=1 python my_experiment.py +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```shell +DD_TRACE_OTEL_ENABLED=1 node my_experiment.js +``` +{{% /tab %}} +{{< /tabs >}} + +### 3. Define evaluators + +Evaluators measure how well your model or agent performs on each record. Both SDKs support function-based evaluators. Python also supports reusable class-based evaluators. + +For detailed information on building evaluators, including the full data model reference and best practices, see the [Evaluation Developer Guide][4]. + +Datadog supports the following evaluator return types: + +- **Boolean**: Returns true or false +- **Score**: Returns a numeric value (float) +- **Categorical**: Returns a labeled category (string) +- **JSON**: Returns structured data (dict or object) + +{{< tabs >}} +{{% tab "Python" %}} +#### Function-based evaluators + +```python +def exact_match(input_data: Dict[str, Any], output_data: str, expected_output: str) -> bool: + return output_data == expected_output + +def overlap(input_data: Dict[str, Any], output_data: str, expected_output: str) -> float: + expected_output_set = set(expected_output) + output_set = set(output_data) + + intersection = len(output_set.intersection(expected_output_set)) + union = len(output_set.union(expected_output_set)) + + return intersection / union + +def fake_llm_as_a_judge(input_data: Dict[str, Any], output_data: str, expected_output: str) -> EvaluatorResult: + fake_llm_call = "excellent" + return EvaluatorResult( + value=fake_llm_call, + reasoning="the model explains itself", + assessment="pass", # or fail + tags={"task": "judge_llm_call"}, + ) + +# Return multiple metrics from one evaluator call +from ddtrace.llmobs import EvaluatorResult, MultiEvaluatorResult + +def multi_metric_evaluator(input_data, output_data, expected_output): + correct = output_data == expected_output + return MultiEvaluatorResult( + { + "correct": EvaluatorResult(value=correct, assessment="pass" if correct else "fail"), + "length": len(str(output_data)), + } + ) + # Emitted as: multi_metric_evaluator-correct, multi_metric_evaluator-length +``` + +#### Class-based evaluators + +```python +from ddtrace.llmobs import BaseEvaluator, EvaluatorContext, EvaluatorResult + +class SemanticSimilarityEvaluator(BaseEvaluator): + def __init__(self, threshold: float = 0.8): + super().__init__(name="semantic_similarity") + self.threshold = threshold + + def evaluate(self, context: EvaluatorContext) -> EvaluatorResult: + score = compute_similarity(context.output_data, context.expected_output) + return EvaluatorResult( + value=score, + reasoning=f"Similarity score: {score:.2f}", + assessment="pass" if score >= self.threshold else "fail", + ) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +The Node.js SDK supports function evaluators. Return a Boolean, number, string, or JSON-serializable object. Use an object map when you want to assign evaluator names explicitly. + +```javascript +function exact_match (inputData, outputData, expectedOutput) { + return outputData === expectedOutput +} + +function overlap (inputData, outputData, expectedOutput) { + const expected = new Set(String(expectedOutput)) + const output = new Set(String(outputData)) + const intersection = [...output].filter(value => expected.has(value)).length + const union = new Set([...expected, ...output]).size + return intersection / union +} + +function fake_llm_as_a_judge (inputData, outputData, expectedOutput) { + // Return a categorical metric. Use a separate evaluator for each metric. + return 'excellent' +} +``` + +Class-based evaluators and `MultiEvaluatorResult` are not part of the Node.js experiments API. Define multiple named functions when you need multiple metrics. +{{% /tab %}} +{{< /tabs >}} + +### 4. (Optional) Define summary evaluators + +Summary evaluators receive aggregated results after all record-level evaluators finish. Use them to calculate dataset-level statistics, such as averages or pass rates. Both SDKs support function-based summary evaluators; Python also supports class-based summary evaluators. + +{{< tabs >}} +{{% tab "Python" %}} +#### Function-based summary evaluators + +```python +def num_exact_matches(inputs, outputs, expected_outputs, evaluators_results): + return evaluators_results["exact_match"].count(True) +``` + +Summary evaluator functions can take a list of any non-null type as `inputs` (string, number, Boolean, object, or array); `outputs` and `expected_outputs` can be lists of any type. `evaluators_results` is a dictionary of lists of results from evaluators, keyed by the name of the evaluator function. + +#### Class-based summary evaluators + +For details on implementing `BaseSummaryEvaluator`, see the [Evaluation Developer Guide](/llm_observability/investigate/evaluations/evaluation_developer_guide/). + +```python +from ddtrace.llmobs import BaseSummaryEvaluator, SummaryEvaluatorContext + +class AverageScoreEvaluator(BaseSummaryEvaluator): + def __init__(self, target_evaluator: str): + super().__init__(name="average_score") + self.target_evaluator = target_evaluator + + def evaluate(self, context: SummaryEvaluatorContext): + scores = context.evaluation_results.get(self.target_evaluator, []) + if not scores: + return None + return sum(scores) / len(scores) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +Summary evaluators are functions that receive the inputs, outputs, expected outputs, and an object containing the results from each evaluator. The optional fifth argument contains record metadata. + +```javascript +function num_exact_matches (inputs, outputs, expectedOutputs, evaluatorResults, metadata) { + return (evaluatorResults.exact_match || []).filter(Boolean).length +} +``` + +The Node.js SDK does not use class-based summary evaluators. Define multiple named summary evaluator functions when you need multiple dataset-level metrics. +{{% /tab %}} +{{< /tabs >}} + +### 5. Create and run the experiment. + +{{< tabs >}} +{{% tab "Python" %}} +```python +experiment = LLMObs.experiment( + name="capital-cities-test", + task=task, + dataset=dataset, + evaluators=[exact_match, overlap, fake_llm_as_a_judge], + summary_evaluators=[num_exact_matches], # optional + description="Testing capital cities knowledge", + config={ + "model_name": "gpt-4", + "version": "1.0" + }, +) + +# Run the experiment +results = experiment.run() # Run on all dataset records + +# Process results +for result in results.get("rows", []): + print(f"Record {result['idx']}") + print(f"Input: {result['input']}") + print(f"Output: {result['output']}") + print(f"Score: {result['evaluations']['evaluator']['value']}") + if result['error']['message']: + print(f"Error: {result['error']['message']}") +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +const experiment = experiments.experiment({ + name: 'capital-cities-test', + task, + dataset, + evaluators: { + exact_match, + overlap, + fake_llm_as_a_judge, + }, + summaryEvaluators: { + num_exact_matches, + }, + description: 'Testing capital cities knowledge', + config: { + model: 'gpt-4', + version: '1.0', + }, +}) + +// Run the experiment. The dataset is pushed automatically if needed. +const results = await experiment.run() + +// Process results +for (const row of results.rows) { + console.log(`Record ${row.index}`) + console.log(`Input: ${JSON.stringify(row.input)}`) + console.log(`Output: ${JSON.stringify(row.output)}`) + console.log(`Score: ${row.evaluations.exact_match}`) + if (row.isError) { + console.log(`Error: ${row.errorMessage}`) + } +} +``` +{{% /tab %}} +{{< /tabs >}} + +Configure experiment execution using the options supported by your SDK: + +{{< tabs >}} +{{% tab "Python" %}} +```python +# Increase execution speed +results = experiment.run(jobs=4) + +# Test your pipeline on a subset of the data +results = experiment.run(sample_size=10) + +# Stop the experiment if an error occurs +results = experiment.run(raise_errors=True) +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +The Node.js SDK runs dataset records sequentially. To manage concurrency, implement it in your task function. To stop the experiment if an error occurs, set `throwOnErrors` to `true`: + +```javascript +const results = await experiment.run({ + throwOnErrors: true, +}) +``` +{{% /tab %}} +{{< /tabs >}} + +### 6. Review your experiment results in Datadog. + +{{< tabs >}} +{{% tab "Python" %}} +```python +print(f"View experiment: {experiment.url}") +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +console.log(`View experiment: ${results.url}`) +// The same URL is available from the experiment after it has run. +console.log(experiment.url()) +``` +{{% /tab %}} +{{< /tabs >}} + +**Note:** LLM Experiments traces are retained for 90 days. + +[1]: /llm_observability/improve/datasets +[4]: /llm_observability/investigate/evaluations/evaluation_developer_guide +[6]: /llm_observability/instrument/otel_instrumentation + +## Further reading + +{{< partial name="whats-next/whats-next.html" >}} diff --git a/hugo/content/en/llm_observability/improve/experiments/api.md b/hugo/content/en/llm_observability/guide/experiments_api.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/api.md rename to hugo/content/en/llm_observability/guide/experiments_api.md index cf99ea1320b..dfff9719533 100644 --- a/hugo/content/en/llm_observability/improve/experiments/api.md +++ b/hugo/content/en/llm_observability/guide/experiments_api.md @@ -1,6 +1,7 @@ --- title: Experiments API aliases: +- /llm_observability/improve/experiments/api/ - /llm_observability/experiments/api/ description: Reference the Agent Observability Experiments HTTP API. --- @@ -558,6 +559,6 @@ Empty body on success. {{% /collapse-content %}} -[1]: /llm_observability/improve/experiments/setup +[1]: /llm_observability/guide/experiments [2]: https://github.com/DataDog/llm-observability/tree/main/experiments [3]: https://www.postman.com/ diff --git a/hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md b/hugo/content/en/llm_observability/guide/prompt_optimization.md similarity index 99% rename from hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md rename to hugo/content/en/llm_observability/guide/prompt_optimization.md index 59e58904473..54c0af953b1 100644 --- a/hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md +++ b/hugo/content/en/llm_observability/guide/prompt_optimization.md @@ -1,11 +1,12 @@ --- title: Prompt Optimization aliases: +- /llm_observability/improve/experiments/prompt_optimization/ - /llm_observability/experiments/prompt_optimization/ description: Automatically improve LLM prompts through iterative refinement and AI-powered evaluation. private: true further_reading: - - link: /llm_observability/improve/experiments/setup + - link: /llm_observability/guide/experiments tag: "Documentation" text: Set up and use Agent Observability Experiments - link: /llm_observability/improve/datasets diff --git a/hugo/content/en/llm_observability/improve/experiments/_index.md b/hugo/content/en/llm_observability/improve/experiments/_index.md index 180f600d0ed..88397955d44 100644 --- a/hugo/content/en/llm_observability/improve/experiments/_index.md +++ b/hugo/content/en/llm_observability/improve/experiments/_index.md @@ -5,7 +5,7 @@ aliases: - /llm_observability/experiments_preview - /llm_observability/experiments/ further_reading: - - link: /llm_observability/improve/experiments/setup + - link: /llm_observability/guide/experiments tag: "Documentation" text: Set up and use Agent Observability Experiments - link: "https://www.datadoghq.com/blog/llm-experiments/" diff --git a/hugo/content/en/llm_observability/improve/experiments/setup.md b/hugo/content/en/llm_observability/improve/experiments/setup.md deleted file mode 100644 index f0d5cbbc9cf..00000000000 --- a/hugo/content/en/llm_observability/improve/experiments/setup.md +++ /dev/null @@ -1,493 +0,0 @@ ---- -title: Setup and Usage -aliases: -- /llm_observability/experiments/setup/ -description: How to set up Agent Observability Experiments and start running experiments. -further_reading: -- link: "https://www.datadoghq.com/blog/debug-and-evaluate-your-ai-app-from-your-coding-agent/" - tag: "Blog" - text: "Debug and evaluate your AI app from your coding agent with Datadog Agent Observability" ---- - -This page describes how to set up and use Agent Observability Experiments with the Python or Node.js SDK. - -{{< tabs >}} -{{% tab "Python" %}} - -## Set up Agent Observability - -If you have not already set up Agent Observability: - -1. Install Agent Observability Python SDK: - - ```shell - pip install ddtrace>=4.3.0 - ``` - -2. Enable Agent Observability: - - ```python - from ddtrace.llmobs import LLMObs - - LLMObs.enable( - api_key="", # defaults to DD_API_KEY environment variable - app_key="", # defaults to DD_APP_KEY environment variable - site="datadoghq.com", # defaults to DD_SITE environment variable - project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set - ) - ``` - -
You must supply both an api_key and app_key.
- -### APM Trace correlation - -To correlate your Experiment spans with [APM Traces][5], run Agent Observability through a Datadog Agent and keep `agentless_enabled` set to `False` (the default). The Agent forwards trace data to APM, which is what enables Experiment ↔ APM Trace correlation. - - ```python - LLMObs.enable( - api_key="", - app_key="", - site="datadoghq.com", - agentless_enabled=False, # default — required for APM Trace correlation - project_name="", - ) - ``` - -If you are running without an Agent (for example, in a notebook or CI environment), you can set `agentless_enabled=True`, but corresponding APM spans are not generated for Experiment spans from agentless runs. - -## Create a project -_Projects_ are the core organizational layer for LLM Experiments. All datasets and experiments live in a project. -You can create a project manually in the Datadog console, API, or SDK by specifying a project name that does not already exist in `LLMObs.enable`. - -```python -LLMObs.enable( - ... - project_name="" # defaults to DD_LLMOBS_PROJECT_NAME environment variable, or "default-project" if the environment variable is not set -) -``` - -## Create a dataset - -A _dataset_ is a collection of _inputs_, and _expected outputs_ and _metadata_ that represent scenarios you want to tests your agent on. Each dataset is associated with a _project_. - -- **input** (required): Represents all the information that the agent can access in a task. -- **expected output** (optional): Also called _ground truth_, represents the ideal answer that the agent should output. You can use _expected output_ to store the actual output of the app, as well as any intermediary results you want to assesss. -- **metadata** (optional): Contains any useful information to categorize the record and use for further analysis. For example: topics, tags, descriptions, notes. - -To create a dataset from a CSV file, use `LLMObs.create_dataset_from_csv()`: - -```python -# Create dataset from CSV -dataset = LLMObs.create_dataset_from_csv( - csv_path="questions.csv", - dataset_name="capitals-of-the-world", - project_name="capitals-project", # Optional: defaults to the project name from LLMObs.enable - description="Geography quiz dataset", # Optional: Dataset description - input_data_columns=["question", "category"], # Columns to use as input - expected_output_columns=["answer"], # Optional: Columns to use as expected output - metadata_columns=["difficulty"], # Optional: Additional columns as metadata - csv_delimiter="," # Optional: Defaults to comma -) - -# Example "questions.csv": -# question,category,answer,difficulty -# What is the capital of Japan?,geography,Tokyo,medium -# What is the capital of Brazil?,geography,Brasília,medium - -``` - -See [Datasets][1] for more information about datasets, including: how to manually create datasets, how to retrieve and manage datasets, and how Datadog retains dataset versions. - -## Create an experiment -An _experiment_ lets you systematically test your LLM application by running your agent across a set of scenarios from your dataset and measuring performance against the expected outputs using evaluators. You can then compare how different app configurations perform, side by side. - -- **task**: Defines the core workflow you want to evaluate. It can range from a single LLM call to a more complex flow involving multiple LLM calls and RAG steps. The task is executed sequentially across all records in the dataset. -- **evaluator**: A function, executed on each record, that measures how well the model or agent performs. Evaluators allow you to compare the output to either the expected output or the original input. - -- **summary evaluators**: Optional functions executed against all the data of the Experiment (input, output, expected, evaluators' results). Summary evaluators allow you to compute more advanced metrics like precision, recall, and accuracy across your dataset. - - -To create an experiment: - - -### 1. Load a dataset - ```python - from ddtrace.llmobs import LLMObs - from typing import Dict, Any, Optional, List - - dataset = LLMObs.pull_dataset("capitals-of-the-world") - ``` - -### 2. Define a task function that processes a single dataset record - - ```python - def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: - question = input_data["question"] - # Your LLM or processing logic here - return "Beijing" if "China" in question else "Unknown" - ``` - A task can take any non-null type as `input_data` (string, number, Boolean, object, array). The output that will be used in the Evaluators can be of any type. - This example generates a string, but a dict can be generated as output to store any intermediary information and compare in the Evaluators. - - Optionally, your task function can accept a third `metadata` parameter to receive the dataset record's metadata: - ```python - def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None, metadata: Optional[Dict[str, Any]] = None) -> str: - difficulty = metadata.get("difficulty", "unknown") if metadata else "unknown" - question = input_data["question"] - return "Beijing" if "China" in question else "Unknown" - ``` - - You can trace the different parts of your Experiment task (workflow, tool calls, etc.) using the [same tracing decorators][2] you use in production. - If you use a [supported framework][3] (OpenAI, Amazon Bedrock, etc.), Agent Observability automatically traces and annotates calls to LLM frameworks and libraries, giving you out-of-the-box observability for calls that your LLM application makes. - - #### Using OpenTelemetry spans inside experiments - - If your application uses [OpenTelemetry instrumentation][6], you can create OTel spans inside your experiment task. With `DD_TRACE_OTEL_ENABLED=1`, ddtrace acts as the OpenTelemetry TracerProvider, so OTel spans appear as children of the experiment span automatically. - - ```python - import json - from opentelemetry import trace - - tracer = trace.get_tracer(__name__) - - def task(input_data: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> str: - question = input_data["question"] - - # OTel gen_ai span — automatically becomes a child of the experiment span - with tracer.start_as_current_span("my-llm-call") as span: - span.set_attribute("gen_ai.operation.name", "chat") - span.set_attribute("gen_ai.system", "openai") - span.set_attribute("gen_ai.request.model", "gpt-4o") - span.set_attribute("gen_ai.usage.input_tokens", 25) - span.set_attribute("gen_ai.usage.output_tokens", 8) - span.set_attribute( - "gen_ai.input.messages", - json.dumps([{"role": "user", "parts": [{"type": "text", "content": question}]}]), - ) - - result = call_my_llm(question) - - span.set_attribute( - "gen_ai.output.messages", - json.dumps([{"role": "assistant", "parts": [{"type": "text", "content": result}]}]), - ) - - return result - ``` - - To enable this, set the `DD_TRACE_OTEL_ENABLED` environment variable: - - ```shell - DD_TRACE_OTEL_ENABLED=1 python my_experiment.py - ``` - - -### 3. Define evaluators - - Evaluators measure how well your model or agent performs on each record. You can define evaluators using two approaches: - - - **Function-based**: Define a function that receives `input_data`, `output_data`, and `expected_output` as separate arguments. Best for one-off evaluators with straightforward logic. - - **Class-based**: Subclass `BaseEvaluator` for reusable evaluators with custom configuration. Class-based evaluators receive an `EvaluatorContext` object with full span context. - - For detailed information on building evaluators, including the full data model reference and best practices, see the [Evaluation Developer Guide][4]. - - Datadog supports the following evaluator return types: - - **Boolean**: returns true or false - - **score**: returns a numeric value (float) - - **categorical**: returns a labeled category (string) - - **json**: returns structured data (dict) - - You can also return: - - An `EvaluatorResult` to capture richer evaluation data, such as `reasoning`, `assessment` (`"pass"` or `"fail"`), `metadata`, and `tags`. - - A `MultiEvaluatorResult` to emit multiple named metrics from a single evaluator call. For details and examples, see the [Evaluation Developer Guide][4]. - - #### Function-based evaluators - - ```python - def exact_match(input_data: Dict[str, Any], output_data: str, expected_output: str) -> bool: - return output_data == expected_output - - def overlap(input_data: Dict[str, Any], output_data: str, expected_output: str) -> float: - expected_output_set = set(expected_output) - output_set = set(output_data) - - intersection = len(output_set.intersection(expected_output_set)) - union = len(output_set.union(expected_output_set)) - - return intersection / union - - def fake_llm_as_a_judge(input_data: Dict[str, Any], output_data: str, expected_output: str) -> EvaluatorResult: - fake_llm_call = "excellent" - return EvaluatorResult( - value=fake_llm_call, - reasoning="the model explains itself", - assessment="pass", # or fail - tags={"task": "judge_llm_call"}, - ) - - # Return multiple metrics from one evaluator call - from ddtrace.llmobs import MultiEvaluatorResult - - def multi_metric_evaluator(input_data, output_data, expected_output): - correct = output_data == expected_output - return MultiEvaluatorResult( - { - "correct": EvaluatorResult(value=correct, assessment="pass" if correct else "fail"), - "length": len(str(output_data)), - } - ) - # Emitted as: multi_metric_evaluator-correct, multi_metric_evaluator-length - ``` - - #### Class-based evaluators - - ```python - from ddtrace.llmobs import BaseEvaluator, EvaluatorContext, EvaluatorResult - - class SemanticSimilarityEvaluator(BaseEvaluator): - def __init__(self, threshold: float = 0.8): - super().__init__(name="semantic_similarity") - self.threshold = threshold - - def evaluate(self, context: EvaluatorContext) -> EvaluatorResult: - score = compute_similarity(context.output_data, context.expected_output) - return EvaluatorResult( - value=score, - reasoning=f"Similarity score: {score:.2f}", - assessment="pass" if score >= self.threshold else "fail", - ) - ``` - -### 4. (Optional) Define summary evaluators - - Summary evaluators run after all record-level evaluators have finished, and receive the aggregated results to compute dataset-level statistics like averages or pass rates. Like record-level evaluators, you can define summary evaluators as functions or classes. - - For the class-based approach using `BaseSummaryEvaluator`, see the [Evaluation Developer Guide][4]. - - #### Function-based summary evaluators - - ```python - def num_exact_matches(inputs, outputs, expected_outputs, evaluators_results): - return evaluators_results["exact_match"].count(True) - - ``` - - Summary evaluator functions can take a list of any non-null type as `inputs` (string, number, Boolean, object, array); `outputs` and `expected_outputs` can be lists of any type. `evaluators_results` is a dictionary of list of results from evaluators, keyed by the name of the evaluator function. For example, in the above code snippet the summary evaluator `num_exact_matches` uses the results (a list of Booleans) from the `exact_match` evaluator to provide a count of number of exact matches. - - #### Class-based summary evaluators - - ```python - from ddtrace.llmobs import BaseSummaryEvaluator, SummaryEvaluatorContext - - class AverageScoreEvaluator(BaseSummaryEvaluator): - def __init__(self, target_evaluator: str): - super().__init__(name="average_score") - self.target_evaluator = target_evaluator - - def evaluate(self, context: SummaryEvaluatorContext): - scores = context.evaluation_results.get(self.target_evaluator, []) - if not scores: - return None - return sum(scores) / len(scores) - ``` - - Datadog supports the following Summary Evaluator return types: - - **Boolean**: returns true or false - - **score**: returns a numeric value (float) - - **categorical**: returns a labeled category (string) - - **json**: returns structured data (dict) - -### 5. Create and run the experiment. - ```python - experiment = LLMObs.experiment( - name="capital-cities-test", - task=task, - dataset=dataset, - evaluators=[exact_match, overlap, fake_llm_as_a_judge], - summary_evaluators=[num_exact_matches], # optional - description="Testing capital cities knowledge", - config={ - "model_name": "gpt-4", - "version": "1.0" - }, - ) - - # Run the experiment - results = experiment.run() # Run on all dataset records - - # Process results - for result in results.get("rows", []): - print(f"Record {result['idx']}") - print(f"Input: {result['input']}") - print(f"Output: {result['output']}") - print(f"Score: {result['evaluations']['evaluator']['value']}") - if result['error']['message']: - print(f"Error: {result['error']['message']}") - ``` - - To increase the execution speed of the experiment, you can enable parallel processing: - ``` - results = experiment.run(jobs=4) - ``` - - To test your pipeline on a subset of the data, use: - ``` - results = experiment.run(sample_size=10) - ``` - - To stop the execution of the Experiment if an error occurs, use: - ``` - results = experiment.run(raise_errors=True) - ``` - -### 6. Review your experiment results in Datadog. - ``` - print(f"View experiment: {experiment.url}") - ``` - -[1]: /llm_observability/improve/datasets -[2]: /llm_observability/instrument/custom_instrumentation?tab=decorators#trace-an-llm-application -[3]: /llm_observability/instrument/auto_instrumentation?tab=python -[4]: /llm_observability/investigate/evaluations/evaluation_developer_guide -[5]: /llm_observability/instrument/agent_observability_and_apm/ -[6]: /llm_observability/instrument/otel_instrumentation - -{{% /tab %}} - -{{% tab "Node.js" %}} - -## Set up Agent Observability - -If you have not already set up Agent Observability: - -1. Install the Agent Observability Node.js SDK: - - ```shell - npm install dd-trace - ``` - -2. Set the API key, application key, and Datadog site: - - ```shell - export DD_API_KEY="" - export DD_APP_KEY="" - export DD_SITE={{< region-param key="dd_site" >}} - ``` - -3. Initialize the tracer with the ML application and Experiments project names: - - ```javascript - const tracer = require('dd-trace').init({ - llmobs: { - mlApp: 'capitals-app', - projectName: 'capitals-project' - } - }) - ``` - -
You must supply both DD_API_KEY and DD_APP_KEY.
- -## Create a project - -Projects contain datasets and experiments. The SDK uses the configured Experiments project name and -creates the project if it does not exist. - -Set `llmobs.projectName` during tracer initialization. You can also set `DD_LLMOBS_PROJECT_NAME`. If neither value is -set, the SDK uses `default-project`. - -## Create a dataset - -Create a dataset with the `tracer.llmobs.experiments` API: - -```javascript -const { experiments } = tracer.llmobs - -const dataset = experiments.createDataset('capitals-of-the-world', { - description: 'Questions and expected capital cities', - records: [ - { - inputData: { question: 'What is the capital of China?' }, - expectedOutput: 'Beijing', - metadata: { difficulty: 'medium' } - }, - { - inputData: { question: 'What is the capital of Japan?' }, - expectedOutput: 'Tokyo', - metadata: { difficulty: 'medium' } - } - ] -}) -``` - -The SDK pushes the dataset when the experiment starts. To use an existing dataset, call -`await experiments.pullDataset('')`. - -## Create an experiment - -An experiment runs a task for each dataset record and evaluates each result. - -### 1. Define a task - -The task receives the record input. It can also receive the experiment configuration and optional record metadata. - -```javascript -function task (inputData) { - const { question } = inputData - - // Add the LLM or agent call to evaluate. - return question.includes('China') ? 'Beijing' : 'Tokyo' -} -``` - -Calls to [supported Node.js integrations](/llm_observability/instrument/auto_instrumentation?tab=nodejs) inside the task -are traced as children of the experiment span. - -### 2. Define evaluators - -An evaluator receives the record input, task output, and expected output. Return a boolean, number, string, or JSON -value to create the corresponding evaluation metric. - -```javascript -function exactMatch (_inputData, outputData, expectedOutput) { - return outputData === expectedOutput -} -``` - -### 3. Create and run the experiment - -```javascript -async function runExperiment () { - const experiment = experiments.experiment({ - name: 'capital-cities-test', - dataset, - task, - evaluators: { - exact_match: exactMatch - }, - description: 'Testing capital cities knowledge', - config: { - modelName: 'gpt-4', - version: '1.0' - } - }) - - const result = await experiment.run() - console.log(`View experiment: ${result.url}`) -} - -runExperiment().catch((error) => { - console.error(error) - process.exitCode = 1 -}) -``` - -The result contains each record output, evaluator score, error, and a URL for the experiment in Datadog. - -{{% /tab %}} -{{< /tabs >}} - -Note: LLM Experiments traces are retained for 90 days. - -## Further reading - -{{< partial name="whats-next/whats-next.html" >}} diff --git a/hugo/content/en/llm_observability/improve/playground.md b/hugo/content/en/llm_observability/improve/playground.md index 4ecb1e63fb0..ee29c5017a4 100644 --- a/hugo/content/en/llm_observability/improve/playground.md +++ b/hugo/content/en/llm_observability/improve/playground.md @@ -10,7 +10,7 @@ further_reading: - link: "/llm_observability/improve/datasets" tag: "Documentation" text: "Datasets" - - link: "/llm_observability/improve/experiments/analyzing_results" + - link: "/llm_observability/guide/analyzing_experiment_results" tag: "Documentation" text: "Analyzing Experiment Results" --- diff --git a/hugo/content/en/llm_observability/instrument/otel_instrumentation.md b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md index 7b5900c8680..a0ac8630089 100644 --- a/hugo/content/en/llm_observability/instrument/otel_instrumentation.md +++ b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md @@ -27,7 +27,7 @@ For information on using Prompt Tracking with OpenTelemetry spans, see [Prompt T ### Experiments -You can use OpenTelemetry spans inside [Agent Observability Experiments](/llm_observability/improve/experiments/setup#using-opentelemetry-spans-inside-experiments). By setting `DD_TRACE_OTEL_ENABLED=1`, OTel spans created inside an experiment task automatically appear as children of the experiment span. +You can use OpenTelemetry spans inside [Agent Observability Experiments](/llm_observability/guide/experiments#using-opentelemetry-spans-inside-experiments). By setting `DD_TRACE_OTEL_ENABLED=1`, OTel spans created inside an experiment task automatically appear as children of the experiment span. ### Multimodal support diff --git a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md index d09b272d7ed..d145f7b594a 100644 --- a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md +++ b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/deepeval.md @@ -109,8 +109,8 @@ After you run an experiment with a DeepEval evaluation, you can view the DeepEva {{< partial name="whats-next/whats-next.html" >}} [1]: /llm_observability/improve/experiments -[2]: /llm_observability/improve/experiments/setup#create-an-experiment +[2]: /llm_observability/guide/experiments#create-an-experiment [3]: https://deepeval.com/docs/metrics-introduction -[4]: /llm_observability/improve/experiments/setup#create-a-dataset +[4]: /llm_observability/guide/experiments#create-a-dataset [5]: /llm_observability/improve/datasets [6]: https://github.com/DataDog/llm-observability/blob/main/experiments/eval-integrations/1-deepeval-demo.py diff --git a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md index a1b08d1a65e..4e973dfd7d1 100644 --- a/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md +++ b/hugo/content/en/llm_observability/investigate/evaluations/external_evaluations/pydantic.md @@ -141,8 +141,8 @@ After you run an experiment with a Pydantic evaluation, you can view the Pydanti {{< partial name="whats-next/whats-next.html" >}} [1]: /llm_observability/improve/experiments -[2]: /llm_observability/improve/experiments/setup#create-an-experiment +[2]: /llm_observability/guide/experiments#create-an-experiment [3]: https://ai.pydantic.dev/evals/ -[4]: /llm_observability/improve/experiments/setup#create-a-dataset +[4]: /llm_observability/guide/experiments#create-a-dataset [5]: /llm_observability/improve/datasets [6]: https://github.com/DataDog/llm-observability/blob/main/experiments/eval-integrations/2-pydantic-demo.py From ec0fcd0ae9c09ce39d0b0b8156b2227e205e69d2 Mon Sep 17 00:00:00 2001 From: mehulsonowal Date: Fri, 9 Oct 2026 12:58:57 -0400 Subject: [PATCH 2/3] [DOCS-15714] Document Node.js evaluator APIs --- .../en/llm_observability/guide/experiments.md | 73 +++++++++++++++++-- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/hugo/content/en/llm_observability/guide/experiments.md b/hugo/content/en/llm_observability/guide/experiments.md index ab2516dccb3..4a39265f3d7 100644 --- a/hugo/content/en/llm_observability/guide/experiments.md +++ b/hugo/content/en/llm_observability/guide/experiments.md @@ -471,7 +471,7 @@ DD_TRACE_OTEL_ENABLED=1 node my_experiment.js ### 3. Define evaluators -Evaluators measure how well your model or agent performs on each record. Both SDKs support function-based evaluators. Python also supports reusable class-based evaluators. +Evaluators measure how well your model or agent performs on each record. Both SDKs support function-based evaluators and reusable class-based evaluators. For detailed information on building evaluators, including the full data model reference and best practices, see the [Evaluation Developer Guide][4]. @@ -543,7 +543,7 @@ class SemanticSimilarityEvaluator(BaseEvaluator): {{% /tab %}} {{% tab "Node.js" %}} -The Node.js SDK supports function evaluators. Return a Boolean, number, string, or JSON-serializable object. Use an object map when you want to assign evaluator names explicitly. +The Node.js SDK supports function and class-based evaluators. Return a Boolean, number, string, or JSON-serializable object. Use an object map when you want to assign evaluator names explicitly. ```javascript function exact_match (inputData, outputData, expectedOutput) { @@ -564,13 +564,58 @@ function fake_llm_as_a_judge (inputData, outputData, expectedOutput) { } ``` -Class-based evaluators and `MultiEvaluatorResult` are not part of the Node.js experiments API. Define multiple named functions when you need multiple metrics. +#### Class-based evaluators + +For reusable evaluators, extend `experiments.BaseEvaluator`. The `evaluate` method receives an `EvaluatorContext` and can return a value or a Promise. Return `experiments.EvaluatorResult` to attach reasoning, assessment, metadata, or tags to the evaluation metric. + +```javascript +const tracer = require('dd-trace').init() +const { experiments } = tracer.llmobs + +class ExactMatchEvaluator extends experiments.BaseEvaluator { + constructor () { + super('exact_match') + } + + evaluate (context) { + const passed = context.outputData === context.expectedOutput + return new experiments.EvaluatorResult(passed, { + assessment: passed ? 'pass' : 'fail', + reasoning: passed ? 'The output matches the expected value.' : 'The output does not match the expected value.', + metadata: { evaluator_version: 'v1' }, + tags: { evaluator_version: 'v1' }, + }) + } +} + +const experiment = experiments.experiment({ + // ...name, dataset, and task... + evaluators: [new ExactMatchEvaluator()], +}) +``` + +#### Managed remote evaluators + +Use `experiments.RemoteEvaluator` to run a custom [LLM-as-a-judge evaluation](/llm_observability/investigate/evaluations/llm_as_a_judge_evaluations/) configured in Datadog. Set `evalName` to the configured evaluation name. The default input mapping includes the task input and output, expected output, record metadata, and experiment span and trace IDs. You can provide `transformFn` to customize the request sent to Datadog. + +```javascript +const remoteEvaluator = new experiments.RemoteEvaluator({ + evalName: '', +}) + +const experiment = experiments.experiment({ + // ...name, dataset, and task... + evaluators: [remoteEvaluator], +}) +``` + +You can mix managed remote evaluators with local function-based or class-based evaluators in the same experiment. {{% /tab %}} {{< /tabs >}} ### 4. (Optional) Define summary evaluators -Summary evaluators receive aggregated results after all record-level evaluators finish. Use them to calculate dataset-level statistics, such as averages or pass rates. Both SDKs support function-based summary evaluators; Python also supports class-based summary evaluators. +Summary evaluators receive aggregated results after all record-level evaluators finish. Use them to calculate dataset-level statistics, such as averages or pass rates. Both SDKs support function-based and class-based summary evaluators. {{< tabs >}} {{% tab "Python" %}} @@ -612,7 +657,25 @@ function num_exact_matches (inputs, outputs, expectedOutputs, evaluatorResults, } ``` -The Node.js SDK does not use class-based summary evaluators. Define multiple named summary evaluator functions when you need multiple dataset-level metrics. +#### Class-based summary evaluators + +For reusable summary evaluators, extend `experiments.BaseSummaryEvaluator`. The `SummaryEvaluatorContext` includes the inputs, outputs, expected outputs, evaluator results, and record metadata. + +```javascript +class PassRateEvaluator extends experiments.BaseSummaryEvaluator { + constructor () { + super('pass_rate') + } + + evaluate (context) { + const values = context.evaluationResults.exact_match || [] + if (values.length === 0) return null + return values.filter(Boolean).length / values.length + } +} +``` + +Add class-based summary evaluators to `summaryEvaluators` alongside function-based evaluators. {{% /tab %}} {{< /tabs >}} From 3bc2cfb8eba031f974147ffaeadb859b1ae82d3e Mon Sep 17 00:00:00 2001 From: mehulsonowal Date: Fri, 9 Oct 2026 17:08:15 -0400 Subject: [PATCH 3/3] [DOCS-15714] Address experiments guide review feedback --- .../en/llm_observability/guide/experiments.md | 39 ++++++++++++------- .../improve/experiments/_index.md | 2 + 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/hugo/content/en/llm_observability/guide/experiments.md b/hugo/content/en/llm_observability/guide/experiments.md index 4a39265f3d7..f62a9f3eae0 100644 --- a/hugo/content/en/llm_observability/guide/experiments.md +++ b/hugo/content/en/llm_observability/guide/experiments.md @@ -10,7 +10,7 @@ further_reading: text: "Debug and evaluate your AI app from your coding agent with Datadog Agent Observability" --- -This guide describes how to set up and use Agent Observability experiments with the Python or Node.js SDK. For complete runnable Node.js examples, see the [Node.js experiments examples](https://github.com/DataDog/llm-observability/tree/main/experiments/nodejs). +This guide describes how to set up and use Agent Observability experiments with the Python or Node.js SDK. See [runnable Node.js experiments examples][7]. ## Prerequisites @@ -92,13 +92,13 @@ DD_LLMOBS_ENABLED=1 DD_LLMOBS_ML_APP= DD_LLMOBS_PROJECT_NAME=< NODE_OPTIONS="--import dd-trace/initialize.mjs" node ``` -For more information, see the [Node.js tracer command-line setup](/llm_observability/instrument/sdk?tab=nodejs#command-line-setup). +For more information, see the [Node.js tracer command-line setup][8]. {{% /tab %}} {{< /tabs >}} -### APM Trace correlation +### APM trace correlation -To correlate experiment spans with [APM traces](/llm_observability/instrument/agent_observability_and_apm/), run Agent Observability through a Datadog Agent. The Agent forwards trace data to APM. +To correlate experiment spans with [APM traces][9], run Agent Observability through a Datadog Agent. The Agent forwards trace data to APM. {{< tabs >}} {{% tab "Python" %}} @@ -365,7 +365,7 @@ def retrieve_context(question): {{% /tab %}} {{% tab "Node.js" %}} -For custom tracing, use `llmobs.trace()`. For automatic instrumentation, see the [supported Node.js frameworks](/llm_observability/instrument/auto_instrumentation?tab=nodejs). +For custom tracing, use `llmobs.trace()`. For automatic instrumentation, see the [supported Node.js frameworks][10]. ```javascript // Example: trace a workflow or tool call with the same LLMObs APIs used in production. @@ -471,7 +471,7 @@ DD_TRACE_OTEL_ENABLED=1 node my_experiment.js ### 3. Define evaluators -Evaluators measure how well your model or agent performs on each record. Both SDKs support function-based evaluators and reusable class-based evaluators. +Evaluators measure how well your model or agent performs on each record. Both SDKs support local evaluators defined as functions or classes. The Node.js SDK also supports managed remote evaluators, which run evaluations configured in Datadog. For detailed information on building evaluators, including the full data model reference and best practices, see the [Evaluation Developer Guide][4]. @@ -543,7 +543,9 @@ class SemanticSimilarityEvaluator(BaseEvaluator): {{% /tab %}} {{% tab "Node.js" %}} -The Node.js SDK supports function and class-based evaluators. Return a Boolean, number, string, or JSON-serializable object. Use an object map when you want to assign evaluator names explicitly. +#### Function-based evaluators + +Return a Boolean, number, string, or JSON-serializable object. Use an object map to assign evaluator names explicitly. ```javascript function exact_match (inputData, outputData, expectedOutput) { @@ -596,7 +598,7 @@ const experiment = experiments.experiment({ #### Managed remote evaluators -Use `experiments.RemoteEvaluator` to run a custom [LLM-as-a-judge evaluation](/llm_observability/investigate/evaluations/llm_as_a_judge_evaluations/) configured in Datadog. Set `evalName` to the configured evaluation name. The default input mapping includes the task input and output, expected output, record metadata, and experiment span and trace IDs. You can provide `transformFn` to customize the request sent to Datadog. +Use `experiments.RemoteEvaluator` to run a custom [LLM-as-a-judge evaluation][11] configured in Datadog. Set `evalName` to the configured evaluation name. The default input mapping includes the task input and output, expected output, record metadata, and experiment span and trace IDs. You can provide `transformFn` to customize the request sent to Datadog. ```javascript const remoteEvaluator = new experiments.RemoteEvaluator({ @@ -621,16 +623,16 @@ Summary evaluators receive aggregated results after all record-level evaluators {{% tab "Python" %}} #### Function-based summary evaluators +Summary evaluator functions can take a list of any non-null type as `inputs` (string, number, Boolean, object, or array); `outputs` and `expected_outputs` can be lists of any type. `evaluators_results` is a dictionary of lists of results from evaluators, keyed by the name of the evaluator function. + ```python def num_exact_matches(inputs, outputs, expected_outputs, evaluators_results): return evaluators_results["exact_match"].count(True) ``` -Summary evaluator functions can take a list of any non-null type as `inputs` (string, number, Boolean, object, or array); `outputs` and `expected_outputs` can be lists of any type. `evaluators_results` is a dictionary of lists of results from evaluators, keyed by the name of the evaluator function. - #### Class-based summary evaluators -For details on implementing `BaseSummaryEvaluator`, see the [Evaluation Developer Guide](/llm_observability/investigate/evaluations/evaluation_developer_guide/). +For details on implementing `BaseSummaryEvaluator`, see the [Evaluation Developer Guide][4]. ```python from ddtrace.llmobs import BaseSummaryEvaluator, SummaryEvaluatorContext @@ -649,7 +651,9 @@ class AverageScoreEvaluator(BaseSummaryEvaluator): {{% /tab %}} {{% tab "Node.js" %}} -Summary evaluators are functions that receive the inputs, outputs, expected outputs, and an object containing the results from each evaluator. The optional fifth argument contains record metadata. +#### Function-based summary evaluators + +Summary evaluator functions receive the inputs, outputs, expected outputs, and an object containing the results from each evaluator. The optional fifth argument contains record metadata. ```javascript function num_exact_matches (inputs, outputs, expectedOutputs, evaluatorResults, metadata) { @@ -679,7 +683,9 @@ Add class-based summary evaluators to `summaryEvaluators` alongside function-bas {{% /tab %}} {{< /tabs >}} -### 5. Create and run the experiment. +### 5. Create and run the experiment + +Create an experiment with your dataset, task, and evaluators, then run it and inspect the results: {{< tabs >}} {{% tab "Python" %}} @@ -776,7 +782,7 @@ const results = await experiment.run({ {{% /tab %}} {{< /tabs >}} -### 6. Review your experiment results in Datadog. +### 6. Review your experiment results in Datadog {{< tabs >}} {{% tab "Python" %}} @@ -799,6 +805,11 @@ console.log(experiment.url()) [1]: /llm_observability/improve/datasets [4]: /llm_observability/investigate/evaluations/evaluation_developer_guide [6]: /llm_observability/instrument/otel_instrumentation +[7]: https://github.com/DataDog/llm-observability/tree/main/experiments/nodejs +[8]: /llm_observability/instrument/sdk?tab=nodejs#command-line-setup +[9]: /llm_observability/instrument/agent_observability_and_apm/ +[10]: /llm_observability/instrument/auto_instrumentation?tab=nodejs +[11]: /llm_observability/investigate/evaluations/llm_as_a_judge_evaluations/ ## Further reading diff --git a/hugo/content/en/llm_observability/improve/experiments/_index.md b/hugo/content/en/llm_observability/improve/experiments/_index.md index 88397955d44..8287b407c5c 100644 --- a/hugo/content/en/llm_observability/improve/experiments/_index.md +++ b/hugo/content/en/llm_observability/improve/experiments/_index.md @@ -38,6 +38,8 @@ further_reading: Agent Observability [Experiments][1] supports the entire lifecycle of building LLM applications and agents. It helps you understand how changes to prompts, models, providers, or system architecture affect performance. With this feature, you can: +
To get started, see the Set up and use Agent Observability Experiments guide.
+ - Create and version datasets - Run and manage experiments - Compare results to evaluate impact