diff --git a/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/build.gradle b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/build.gradle new file mode 100644 index 00000000..a58752b1 --- /dev/null +++ b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/build.gradle @@ -0,0 +1,62 @@ +def awsBedrockVersion = '2.30.0' + +muzzle { + pass { + group = 'software.amazon.awssdk' + module = 'bedrockruntime' + // num of aws patch versions is huge so list out minor versions instead + pinVersions '2.30.0', '2.31.0', '2.32.0', '2.33.0', '2.34.0', + '2.35.0', '2.36.0', '2.37.0', '2.38.0', '2.39.0', + '2.40.0', '2.41.0' + // sdk-core is listed explicitly because SdkDefaultClientBuilder (the ByteBuddy + // interception target) lives there, not in bedrockruntime itself + extraDependency 'software.amazon.awssdk:sdk-core' + extraDependency "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + extraDependency "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}" + extraDependency "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jacksonVersion}" + } + pass { + group = 'software.amazon.awssdk' + module = 'bedrockruntime' + versions = '[2.42.36,)' // latest version and up + extraDependency 'software.amazon.awssdk:sdk-core' + extraDependency "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + extraDependency "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}" + extraDependency "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jacksonVersion}" + } +} + +dependencies { + compileOnly project(':braintrust-java-agent:instrumenter') + implementation "io.opentelemetry:opentelemetry-api:${otelVersion}" + implementation 'com.google.code.findbugs:jsr305:3.0.2' + implementation "org.slf4j:slf4j-api:${slf4jVersion}" + implementation project(':braintrust-sdk') + + // ByteBuddy for ElementMatcher types used in instrumentation definitions + compileOnly 'net.bytebuddy:byte-buddy:1.17.5' + + // Target library — compileOnly because it will be on the app classpath at runtime + compileOnly "software.amazon.awssdk:bedrockruntime:${awsBedrockVersion}" + + // Test dependencies + testImplementation(testFixtures(project(":test-harness"))) + testImplementation project(':braintrust-java-agent:instrumenter') + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testImplementation 'net.bytebuddy:byte-buddy-agent:1.17.5' + testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}" + testImplementation "software.amazon.awssdk:bedrockruntime:${awsBedrockVersion}" + testImplementation "software.amazon.awssdk:netty-nio-client:${awsBedrockVersion}" + testImplementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" +} + +test { + useJUnitPlatform() + workingDir = rootProject.projectDir + testLogging { + events "passed", "skipped", "failed" + showStandardStreams = true + exceptionFormat "full" + } +} diff --git a/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustAWSBedrock.java b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustAWSBedrock.java new file mode 100644 index 00000000..7d5716fa --- /dev/null +++ b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustAWSBedrock.java @@ -0,0 +1,71 @@ +package dev.braintrust.instrumentation.awsbedrock.v2_30_0; + +import io.opentelemetry.api.OpenTelemetry; +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClientBuilder; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClientBuilder; + +/** Braintrust instrumentation for the AWS Bedrock Runtime client. */ +@Slf4j +public class BraintrustAWSBedrock { + + /** + * Wraps a {@link BedrockRuntimeClientBuilder} so that every {@code converse} call made through + * the resulting client is traced via OpenTelemetry. + * + *

Call this method after applying all custom builder settings + * + * @param openTelemetry the OpenTelemetry instance to use for tracing + * @param builder the client builder to instrument + * @return the same builder (for fluent chaining) + */ + public static BedrockRuntimeClientBuilder wrap( + OpenTelemetry openTelemetry, BedrockRuntimeClientBuilder builder) { + try { + // Read existing config so we don't clobber user-registered interceptors + ClientOverrideConfiguration existing = builder.overrideConfiguration(); + ClientOverrideConfiguration.Builder configBuilder = existing.toBuilder(); + for (var interceptor : configBuilder.executionInterceptors()) { + if (interceptor instanceof BraintrustBedrockInterceptor) { + log.info("builder already wrapped. Skipping"); + return builder; + } + } + configBuilder.addExecutionInterceptor(new BraintrustBedrockInterceptor(openTelemetry)); + builder.overrideConfiguration(configBuilder.build()); + } catch (Exception e) { + log.warn("Failed to apply Bedrock instrumentation", e); + } + return builder; + } + + /** + * Wraps a {@link BedrockRuntimeAsyncClientBuilder} so that every {@code converseStream} call + * made through the resulting client is traced via OpenTelemetry. + * + *

Call this method after applying all custom builder settings. + * + * @param openTelemetry the OpenTelemetry instance to use for tracing + * @param builder the async client builder to instrument + * @return the same builder (for fluent chaining) + */ + public static BedrockRuntimeAsyncClientBuilder wrap( + OpenTelemetry openTelemetry, BedrockRuntimeAsyncClientBuilder builder) { + try { + ClientOverrideConfiguration existing = builder.overrideConfiguration(); + ClientOverrideConfiguration.Builder configBuilder = existing.toBuilder(); + for (var interceptor : configBuilder.executionInterceptors()) { + if (interceptor instanceof BraintrustBedrockInterceptor) { + log.info("async builder already wrapped. Skipping"); + return builder; + } + } + configBuilder.addExecutionInterceptor(new BraintrustBedrockInterceptor(openTelemetry)); + builder.overrideConfiguration(configBuilder.build()); + } catch (Exception e) { + log.warn("Failed to apply async Bedrock instrumentation", e); + } + return builder; + } +} diff --git a/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustBedrockInterceptor.java b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustBedrockInterceptor.java new file mode 100644 index 00000000..6f3e0d58 --- /dev/null +++ b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustBedrockInterceptor.java @@ -0,0 +1,412 @@ +package dev.braintrust.instrumentation.awsbedrock.v2_30_0; + +import dev.braintrust.instrumentation.InstrumentationSemConv; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.reactivestreams.Publisher; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context.BeforeExecution; +import software.amazon.awssdk.core.interceptor.Context.ModifyHttpRequest; +import software.amazon.awssdk.core.interceptor.Context.ModifyHttpResponse; +import software.amazon.awssdk.core.interceptor.ExecutionAttribute; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest; +import software.amazon.awssdk.thirdparty.jackson.core.JsonFactory; +import software.amazon.awssdk.thirdparty.jackson.core.JsonParser; +import software.amazon.awssdk.thirdparty.jackson.core.JsonToken; +import software.amazon.eventstream.Message; +import software.amazon.eventstream.MessageDecoder; + +/** + * AWS SDK ExecutionInterceptor that creates OpenTelemetry spans for Bedrock Converse calls, + * capturing the raw request and response bodies via {@link InstrumentationSemConv}. + */ +@Slf4j +class BraintrustBedrockInterceptor implements ExecutionInterceptor { + private static final String INSTRUMENTATION_NAME = "braintrust-aws-bedrock"; + + private static final ExecutionAttribute SPAN_ATTRIBUTE = + new ExecutionAttribute<>("braintrust.span"); + private static final ExecutionAttribute MODEL_ID_ATTRIBUTE = + new ExecutionAttribute<>("braintrust.modelId"); + + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + + private final Tracer tracer; + + BraintrustBedrockInterceptor(OpenTelemetry openTelemetry) { + this.tracer = openTelemetry.getTracer(INSTRUMENTATION_NAME); + } + + private static final Set INSTRUMENTED_OPERATIONS = Set.of("Converse", "ConverseStream"); + + @Override + public void beforeExecution(BeforeExecution context, ExecutionAttributes executionAttributes) { + String operationName = + executionAttributes.getAttribute(SdkExecutionAttribute.OPERATION_NAME); + + // Only instrument Converse and ConverseStream — other Bedrock operations + // (InvokeModel, ApplyGuardrail, etc.) are not LLM calls we know how to tag. + if (!INSTRUMENTED_OPERATIONS.contains(operationName)) { + return; + } + + SdkRequest sdkRequest = context.request(); + String modelId = extractModelId(sdkRequest); + + Span span = tracer.spanBuilder(operationName).setParent(Context.current()).startSpan(); + executionAttributes.putAttribute(SPAN_ATTRIBUTE, span); + if (modelId != null) { + executionAttributes.putAttribute(MODEL_ID_ATTRIBUTE, modelId); + } + } + + @Override + public SdkHttpRequest modifyHttpRequest( + ModifyHttpRequest context, ExecutionAttributes executionAttributes) { + Span span = executionAttributes.getAttribute(SPAN_ATTRIBUTE); + if (span == null) { + return context.httpRequest(); + } + + SdkHttpRequest httpRequest = context.httpRequest(); + String modelId = executionAttributes.getAttribute(MODEL_ID_ATTRIBUTE); + + if (modelId == null) { + modelId = extractModelIdFromPath(httpRequest.encodedPath()); + if (modelId != null) { + executionAttributes.putAttribute(MODEL_ID_ATTRIBUTE, modelId); + } + } + + String requestBody = null; + if (context.requestBody().isPresent()) { + try (InputStream is = context.requestBody().get().contentStreamProvider().newStream()) { + requestBody = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } catch (Exception e) { + log.debug("Failed to capture request body", e); + } + } + + String baseUrl = httpRequest.protocol() + "://" + httpRequest.host(); + List pathSegments = + Arrays.stream(httpRequest.encodedPath().split("/")) + .filter(s -> !s.isEmpty()) + .toList(); + + InstrumentationSemConv.tagLLMSpanRequest( + span, + InstrumentationSemConv.PROVIDER_NAME_BEDROCK, + baseUrl, + pathSegments, + "POST", + requestBody, + modelId); + + return httpRequest; + } + + @Override + public Optional modifyHttpResponseContent( + ModifyHttpResponse context, ExecutionAttributes executionAttributes) { + Span span = executionAttributes.getAttribute(SPAN_ATTRIBUTE); + if (span == null) { + return context.responseBody(); + } + + // Only intercept successful responses. On 4xx/5xx the SDK needs to read the body + // itself to parse the AWS error message — consuming it here would swallow that. + if (context.httpResponse().statusCode() >= 300) { + return context.responseBody(); + } + + Optional body = context.responseBody(); + if (body.isPresent()) { + try { + final byte[] bytes = body.get().readAllBytes(); + try { + String responseBodyStr = new String(bytes, StandardCharsets.UTF_8); + InstrumentationSemConv.tagLLMSpanResponse( + span, InstrumentationSemConv.PROVIDER_NAME_BEDROCK, responseBodyStr); + } catch (Exception e) { + log.debug("Failed to capture response body", e); + } + return Optional.of(new ByteArrayInputStream(bytes)); + } catch (IOException e) { + // unlikely this will happen, but if we get here there's no sensible recovery + throw new RuntimeException("failed to ready response body bytes", e); + } + } + return body; + } + + /** + * Intercepts the async response stream for {@code converseStream} calls. Tees the reactive + * {@link Publisher} so that bytes are fed to a {@link MessageDecoder} as they arrive, and on + * completion the decoded event-stream frames are used to tag the span. + */ + @Override + public Optional> modifyAsyncHttpResponseContent( + software.amazon.awssdk.core.interceptor.Context.ModifyHttpResponse context, + ExecutionAttributes executionAttributes) { + Span span = executionAttributes.getAttribute(SPAN_ATTRIBUTE); + Optional> publisherOpt = context.responsePublisher(); + if (span == null || publisherOpt.isEmpty()) { + return publisherOpt; + } + + // Only intercept successful responses — error responses must flow through untouched + // so the SDK can parse the AWS error body. + if (context.httpResponse().statusCode() >= 300) { + return publisherOpt; + } + + Publisher original = publisherOpt.get(); + Publisher teed = + subscriber -> original.subscribe(new TeeingSubscriber(subscriber, span)); + return Optional.of(teed); + } + + @Override + public void afterExecution( + software.amazon.awssdk.core.interceptor.Context.AfterExecution context, + ExecutionAttributes executionAttributes) { + endSpan(executionAttributes, null); + } + + @Override + public void onExecutionFailure( + software.amazon.awssdk.core.interceptor.Context.FailedExecution context, + ExecutionAttributes executionAttributes) { + endSpan(executionAttributes, context.exception()); + } + + private static void endSpan( + ExecutionAttributes executionAttributes, @javax.annotation.Nullable Throwable error) { + Span span = executionAttributes.getAttribute(SPAN_ATTRIBUTE); + if (span == null) { + return; + } + if (error != null) { + InstrumentationSemConv.tagLLMSpanResponse(span, error); + } + span.end(); + } + + private static String extractModelId(SdkRequest request) { + if (request instanceof ConverseRequest r) return r.modelId(); + if (request instanceof ConverseStreamRequest r) return r.modelId(); + return null; + } + + private static String extractModelIdFromPath(String path) { + if (path != null && path.startsWith("/model/")) { + int start = "/model/".length(); + int end = path.indexOf("/", start); + if (end > start) { + return java.net.URLDecoder.decode( + path.substring(start, end), StandardCharsets.UTF_8); + } + } + return null; + } + + /** + * Tees the reactive byte stream into a {@link MessageDecoder}. On completion, decodes each + * event-stream frame using the AWS SDK's shaded Jackson streaming parser, accumulates the + * response content, and hands a synthetic Converse-shaped JSON string to semconv. + */ + private static class TeeingSubscriber implements Subscriber { + private final Subscriber downstream; + private final Span span; + private final MessageDecoder decoder = new MessageDecoder(); + + // Accumulated incrementally in onNext — no message list retained. + private final StringBuilder text = new StringBuilder(); + private String stopReason = null; + private int inputTokens = 0; + private int outputTokens = 0; + private long startNanos; + private Long timeToFirstTokenNanos = null; + + TeeingSubscriber(Subscriber downstream, Span span) { + this.downstream = downstream; + this.span = span; + } + + @Override + public void onSubscribe(Subscription s) { + startNanos = System.nanoTime(); + downstream.onSubscribe(s); + } + + @Override + public void onNext(ByteBuffer buf) { + byte[] copy = new byte[buf.remaining()]; + buf.duplicate().get(copy); + try { + decoder.feed(copy); + for (Message msg : decoder.getDecodedMessages()) { + var h = msg.getHeaders().get(":event-type"); + if (h == null) continue; + String eventType = h.getString(); + byte[] payload = msg.getPayload(); + switch (eventType) { + case "contentBlockDelta" -> { + String t = parseDeltaText(payload); + if (t != null) { + text.append(t); + if (timeToFirstTokenNanos == null) { + timeToFirstTokenNanos = System.nanoTime() - startNanos; + } + } + } + case "messageStop" -> stopReason = parseStopReason(payload); + case "metadata" -> { + int[] tokens = parseTokenUsage(payload); + inputTokens = tokens[0]; + outputTokens = tokens[1]; + } + default -> {} + } + } + } catch (Exception e) { + log.debug("Failed to feed event-stream decoder", e); + } + downstream.onNext(buf); + } + + @Override + public void onError(Throwable t) { + downstream.onError(t); + } + + @Override + public void onComplete() { + try { + InstrumentationSemConv.tagLLMSpanResponse( + span, + InstrumentationSemConv.PROVIDER_NAME_BEDROCK, + buildConverseJson(text.toString(), stopReason, inputTokens, outputTokens), + timeToFirstTokenNanos); + } catch (Exception e) { + log.debug("Failed to tag span from streaming response", e); + } finally { + downstream.onComplete(); + } + } + + /** + * Parses {@code delta.text} from a {@code contentBlockDelta} payload: {@code + * {"contentBlockIndex":0,"delta":{"text":"...","type":"text_delta"}}} + */ + private static String parseDeltaText(byte[] payload) throws Exception { + try (JsonParser p = JSON_FACTORY.createParser(payload)) { + boolean inDelta = false; + while (p.nextToken() != null) { + if (p.currentToken() == JsonToken.FIELD_NAME) { + if ("delta".equals(p.currentName())) { + inDelta = true; + } else if (inDelta && "text".equals(p.currentName())) { + p.nextToken(); + return p.getText(); + } + } else if (p.currentToken() == JsonToken.END_OBJECT) { + inDelta = false; + } + } + } + return null; + } + + /** + * Parses {@code stopReason} from a {@code messageStop} payload: {@code + * {"stopReason":"end_turn"}} + */ + private static String parseStopReason(byte[] payload) throws Exception { + try (JsonParser p = JSON_FACTORY.createParser(payload)) { + while (p.nextToken() != null) { + if (p.currentToken() == JsonToken.FIELD_NAME + && "stopReason".equals(p.currentName())) { + p.nextToken(); + return p.getText(); + } + } + } + return null; + } + + /** + * Parses {@code [inputTokens, outputTokens]} from a {@code metadata} payload: {@code + * {"usage":{"inputTokens":N,"outputTokens":M},"metrics":{...}}} + */ + private static int[] parseTokenUsage(byte[] payload) throws Exception { + int inputTokens = 0; + int outputTokens = 0; + try (JsonParser p = JSON_FACTORY.createParser(payload)) { + while (p.nextToken() != null) { + if (p.currentToken() == JsonToken.FIELD_NAME) { + if ("inputTokens".equals(p.currentName())) { + p.nextToken(); + inputTokens = p.getIntValue(); + } else if ("outputTokens".equals(p.currentName())) { + p.nextToken(); + outputTokens = p.getIntValue(); + } + } + } + } + return new int[] {inputTokens, outputTokens}; + } + + /** + * Builds a synthetic Converse-shaped JSON string matching what {@code tagBedrockResponse} + * expects, using the shaded Jackson generator for correct escaping. + */ + private static String buildConverseJson( + String text, String stopReason, int inputTokens, int outputTokens) + throws Exception { + StringWriter sw = new StringWriter(); + try (var gen = JSON_FACTORY.createGenerator(sw)) { + gen.writeStartObject(); + gen.writeObjectFieldStart("output"); + gen.writeObjectFieldStart("message"); + gen.writeStringField("role", "assistant"); + gen.writeArrayFieldStart("content"); + gen.writeStartObject(); + gen.writeStringField("text", text); + gen.writeEndObject(); + gen.writeEndArray(); + gen.writeEndObject(); // message + gen.writeEndObject(); // output + gen.writeStringField("stopReason", stopReason != null ? stopReason : "end_turn"); + gen.writeObjectFieldStart("usage"); + gen.writeNumberField("inputTokens", inputTokens); + gen.writeNumberField("outputTokens", outputTokens); + gen.writeNumberField("totalTokens", inputTokens + outputTokens); + gen.writeEndObject(); // usage + gen.writeEndObject(); + } + return sw.toString(); + } + } +} diff --git a/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/auto/AWSBedrockInstrumentationModule.java b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/auto/AWSBedrockInstrumentationModule.java new file mode 100644 index 00000000..3b32edc8 --- /dev/null +++ b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/main/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/auto/AWSBedrockInstrumentationModule.java @@ -0,0 +1,83 @@ +package dev.braintrust.instrumentation.awsbedrock.v2_30_0.auto; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import dev.braintrust.instrumentation.InstrumentationModule; +import dev.braintrust.instrumentation.TypeInstrumentation; +import dev.braintrust.instrumentation.TypeTransformer; +import dev.braintrust.instrumentation.awsbedrock.v2_30_0.BraintrustAWSBedrock; +import io.opentelemetry.api.GlobalOpenTelemetry; +import java.util.List; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClientBuilder; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClientBuilder; + +/** + * Auto-instruments the AWS Bedrock Runtime sync and async client builders by hooking into {@code + * SdkDefaultClientBuilder.build()} — the single {@code final} method in the AWS SDK builder + * hierarchy that both {@link BedrockRuntimeClientBuilder} and {@link + * BedrockRuntimeAsyncClientBuilder} ultimately call. The advice checks the runtime type of {@code + * this} to limit instrumentation to Bedrock builders only. + */ +@AutoService(InstrumentationModule.class) +public class AWSBedrockInstrumentationModule extends InstrumentationModule { + private static final String MANUAL_INSTRUMENTATION_PACKAGE = + "dev.braintrust.instrumentation.awsbedrock.v2_30_0."; + + public AWSBedrockInstrumentationModule() { + super("aws_bedrock_2_30_0"); + } + + @Override + public List getHelperClassNames() { + return List.of( + MANUAL_INSTRUMENTATION_PACKAGE + "BraintrustAWSBedrock", + MANUAL_INSTRUMENTATION_PACKAGE + "BraintrustBedrockInterceptor", + MANUAL_INSTRUMENTATION_PACKAGE + "BraintrustBedrockInterceptor$TeeingSubscriber", + "dev.braintrust.json.BraintrustJsonMapper", + "dev.braintrust.instrumentation.InstrumentationSemConv"); + } + + @Override + public List typeInstrumentations() { + return List.of(new SdkDefaultClientBuilderInstrumentation()); + } + + /** + * Targets {@code SdkDefaultClientBuilder} — the abstract base that defines the {@code final + * build()} method inherited by all AWS SDK client builders, including both Bedrock variants. + */ + public static class SdkDefaultClientBuilderInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named("software.amazon.awssdk.core.client.builder.SdkDefaultClientBuilder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + AWSBedrockInstrumentationModule.class.getName() + "$BedrockBuilderAdvice"); + } + } + + /** + * Fires on entry to {@code build()} for any AWS SDK client builder. Uses {@code instanceof} + * checks to limit actual work to Bedrock builders, then calls the idempotent {@code wrap()} + * method to register the Braintrust {@code ExecutionInterceptor} before the client is built. + */ + public static class BedrockBuilderAdvice { + @Advice.OnMethodEnter + public static void build(@Advice.This Object builder) { + if (builder instanceof BedrockRuntimeClientBuilder bedrockBuilder) { + BraintrustAWSBedrock.wrap(GlobalOpenTelemetry.get(), bedrockBuilder); + } else if (builder instanceof BedrockRuntimeAsyncClientBuilder bedrockBuilder) { + BraintrustAWSBedrock.wrap(GlobalOpenTelemetry.get(), bedrockBuilder); + } + } + } +} diff --git a/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/test/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustAWSBedrockTest.java b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/test/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustAWSBedrockTest.java new file mode 100644 index 00000000..4876436a --- /dev/null +++ b/braintrust-sdk/instrumentation/aws_bedrock_2_30_0/src/test/java/dev/braintrust/instrumentation/awsbedrock/v2_30_0/BraintrustAWSBedrockTest.java @@ -0,0 +1,152 @@ +package dev.braintrust.instrumentation.awsbedrock.v2_30_0; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.Bedrock30TestUtils; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.Instrumenter; +import io.opentelemetry.api.common.AttributeKey; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.bedrockruntime.model.ContentBlock; +import software.amazon.awssdk.services.bedrockruntime.model.ConversationRole; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler; +import software.amazon.awssdk.services.bedrockruntime.model.Message; + +public class BraintrustAWSBedrockTest { + + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + @BeforeAll + static void beforeAll() { + var instrumentation = ByteBuddyAgent.install(); + Instrumenter.install(instrumentation, BraintrustAWSBedrockTest.class.getClassLoader()); + } + + private TestHarness testHarness; + private Bedrock30TestUtils bedrockUtils; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + bedrockUtils = new Bedrock30TestUtils(testHarness); + } + + static Stream modelProvider() { + return Stream.of( + Arguments.of("us.anthropic.claude-3-haiku-20240307-v1:0"), + Arguments.of("us.amazon.nova-lite-v1:0")); + } + + @ParameterizedTest(name = "converse with {0}") + @MethodSource("modelProvider") + @SneakyThrows + void converseProducesLlmSpan(String modelId) { + try (var client = bedrockUtils.syncClientBuilder().build()) { + var response = + client.converse( + ConverseRequest.builder() + .modelId(modelId) + .messages( + Message.builder() + .role(ConversationRole.USER) + .content( + ContentBlock.fromText( + "What is the capital of France?" + + " Reply in one word.")) + .build()) + .build()); + assertNotNull(response); + assertFalse( + response.output().message().content().isEmpty(), + "response should have content"); + + var spans = testHarness.awaitExportedSpans(1); + assertEquals(1, spans.size(), "expected exactly one span"); + var span = spans.get(0); + + String spanAttributesJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(spanAttributesJson, "braintrust.span_attributes should be set"); + JsonNode spanAttributes = JSON_MAPPER.readTree(spanAttributesJson); + assertEquals("llm", spanAttributes.get("type").asText()); + assertNotNull( + span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json")), + "braintrust.input_json should be set"); + assertNotNull( + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")), + "braintrust.output_json should be set"); + } + } + + @Test + @SneakyThrows + void converseStreamProducesLlmSpan() { + String modelId = "us.anthropic.claude-3-haiku-20240307-v1:0"; + + try (var client = bedrockUtils.asyncClientBuilder().build()) { + var accumulatedText = new AtomicReference<>(new StringBuilder()); + + var responseHandler = + ConverseStreamResponseHandler.builder() + .subscriber( + ConverseStreamResponseHandler.Visitor.builder() + .onContentBlockDelta( + evt -> { + if (evt.delta().text() != null) { + accumulatedText + .get() + .append(evt.delta().text()); + } + }) + .build()) + .build(); + + client.converseStream( + ConverseStreamRequest.builder() + .modelId(modelId) + .messages( + Message.builder() + .role(ConversationRole.USER) + .content( + ContentBlock.fromText( + "What is the capital of France?" + + " Reply in one word.")) + .build()) + .build(), + responseHandler) + .get(); + + assertFalse( + accumulatedText.get().isEmpty(), "should have received streamed text content"); + + var spans = testHarness.awaitExportedSpans(1); + assertEquals(1, spans.size(), "expected exactly one span"); + var span = spans.get(0); + + String spanAttributesJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(spanAttributesJson, "braintrust.span_attributes should be set"); + JsonNode spanAttributes = JSON_MAPPER.readTree(spanAttributesJson); + assertEquals("llm", spanAttributes.get("type").asText()); + assertNotNull( + span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json")), + "braintrust.input_json should be set"); + assertNotNull( + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")), + "braintrust.output_json should be set"); + } + } +} diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java index 6f47587c..05c04afd 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java @@ -17,6 +17,7 @@ public class InstrumentationSemConv { public static final String PROVIDER_NAME_OPENAI = "openai"; public static final String PROVIDER_NAME_ANTHROPIC = "anthropic"; + public static final String PROVIDER_NAME_BEDROCK = "bedrock"; public static final String PROVIDER_NAME_OTHER = "generic-ai-provider"; public static final String UNSET_LLM_SPAN_NAME = "llm"; @@ -32,6 +33,25 @@ public static void tagLLMSpanRequest( @Nonnull List pathSegments, @Nonnull String method, @Nullable String requestBody) { + tagLLMSpanRequest(span, providerName, baseUrl, pathSegments, method, requestBody, null); + } + + /** + * Tag a span with LLM request metadata. + * + * @param modelId explicit model identifier — used by providers (e.g. Bedrock) where the model + * is not present in the request body. When {@code null} the model is extracted from the + * request body if possible. + */ + @SneakyThrows + public static void tagLLMSpanRequest( + Span span, + @Nonnull String providerName, + @Nonnull String baseUrl, + @Nonnull List pathSegments, + @Nonnull String method, + @Nullable String requestBody, + @Nullable String modelId) { switch (providerName) { case PROVIDER_NAME_OPENAI -> tagOpenAIRequest( @@ -39,6 +59,15 @@ public static void tagLLMSpanRequest( case PROVIDER_NAME_ANTHROPIC -> tagAnthropicRequest( span, providerName, baseUrl, pathSegments, method, requestBody); + case PROVIDER_NAME_BEDROCK -> + tagBedrockRequest( + span, + providerName, + baseUrl, + pathSegments, + method, + requestBody, + modelId); default -> tagOpenAIRequest( span, providerName, baseUrl, pathSegments, method, requestBody); @@ -61,6 +90,8 @@ public static void tagLLMSpanResponse( tagOpenAIResponse(span, responseBody, timeToFirstTokenNanoseconds); case PROVIDER_NAME_ANTHROPIC -> tagAnthropicResponse(span, responseBody, timeToFirstTokenNanoseconds); + case PROVIDER_NAME_BEDROCK -> + tagBedrockResponse(span, responseBody, timeToFirstTokenNanoseconds); default -> tagOpenAIResponse(span, responseBody, timeToFirstTokenNanoseconds); } } @@ -234,6 +265,102 @@ private static void tagAnthropicResponse( } } + // ------------------------------------------------------------------------- + // AWS Bedrock provider implementation + // ------------------------------------------------------------------------- + + @SneakyThrows + private static void tagBedrockRequest( + Span span, + String providerName, + String baseUrl, + List pathSegments, + String method, + @Nullable String requestBody, + @Nullable String modelId) { + String endpoint = bedrockEndpoint(pathSegments); + span.updateName("bedrock." + endpoint); + span.setAttribute("braintrust.span_attributes", toJson(Map.of("type", "llm"))); + + Map metadata = new HashMap<>(); + metadata.put("provider", "bedrock"); + metadata.put("endpoint", endpoint); + metadata.put("request_path", String.join("/", pathSegments)); + metadata.put("request_base_uri", baseUrl); + metadata.put("request_method", method); + + if (modelId != null) { + metadata.put("model", modelId); + } + + if (requestBody != null) { + JsonNode requestJson = BraintrustJsonMapper.get().readTree(requestBody); + // Extract inference parameters from inferenceConfig + if (requestJson.has("inferenceConfig")) { + JsonNode cfg = requestJson.get("inferenceConfig"); + if (cfg.has("maxTokens")) metadata.put("max_tokens", cfg.get("maxTokens")); + if (cfg.has("temperature")) metadata.put("temperature", cfg.get("temperature")); + if (cfg.has("topP")) metadata.put("top_p", cfg.get("topP")); + if (cfg.has("stopSequences")) + metadata.put("stop_sequences", cfg.get("stopSequences")); + } + // Bedrock Converse uses "messages" with typed content block arrays like + // [{"text":"..."}] + if (requestJson.has("messages")) { + ArrayNode inputArray = BraintrustJsonMapper.get().createArrayNode(); + // Bedrock puts system prompts in a separate top-level "system" array: + // [{"text": "..."}]. Prepend as a synthetic {role:"system", content:[...]} entry. + if (requestJson.has("system") + && requestJson.get("system").isArray() + && !requestJson.get("system").isEmpty()) { + var systemNode = BraintrustJsonMapper.get().createObjectNode(); + systemNode.put("role", "system"); + systemNode.set("content", requestJson.get("system")); + inputArray.add(systemNode); + } + for (JsonNode msg : requestJson.get("messages")) { + inputArray.add(normalizeBedrockMessage(msg)); + } + span.setAttribute("braintrust.input_json", toJson(inputArray)); + } + } + + span.setAttribute("braintrust.metadata", toJson(metadata)); + } + + @SneakyThrows + private static void tagBedrockResponse( + Span span, String responseBody, @Nullable Long timeToFirstTokenNanoseconds) { + JsonNode responseJson = BraintrustJsonMapper.get().readTree(responseBody); + + // Bedrock output lives at output.message. Normalize to a single-element array matching the + // same [{role, content: [...]}] shape as input so the UI can render the LLM thread view. + if (responseJson.has("output") && responseJson.get("output").has("message")) { + JsonNode message = responseJson.get("output").get("message"); + ArrayNode outputArray = BraintrustJsonMapper.get().createArrayNode(); + outputArray.add(normalizeBedrockMessage(message)); + span.setAttribute("braintrust.output_json", toJson(outputArray)); + } + + Map metrics = new HashMap<>(); + if (timeToFirstTokenNanoseconds != null) { + metrics.put("time_to_first_token", timeToFirstTokenNanoseconds / 1_000_000_000.0); + } + + // Bedrock usage uses camelCase: inputTokens, outputTokens, totalTokens + if (responseJson.has("usage")) { + JsonNode usage = responseJson.get("usage"); + if (usage.has("inputTokens")) metrics.put("prompt_tokens", usage.get("inputTokens")); + if (usage.has("outputTokens")) + metrics.put("completion_tokens", usage.get("outputTokens")); + if (usage.has("totalTokens")) metrics.put("tokens", usage.get("totalTokens")); + } + + if (!metrics.isEmpty()) { + span.setAttribute("braintrust.metrics", toJson(metrics)); + } + } + // ------------------------------------------------------------------------- // Shared helpers // ------------------------------------------------------------------------- @@ -263,6 +390,57 @@ private static JsonNode simplifyAnthropicMessage(JsonNode msg) { return msg; } + /** + * Normalizes a Bedrock Converse message so its content blocks are compatible with the UI's + * schema checks. The Converse wire format uses {@code {"text":"..."}} for text blocks, but both + * the OpenAI and Anthropic schemas the UI validates against require an explicit {@code + * "type":"text"} field. This method adds {@code "type"} to any content block that has a + * recognized Bedrock key but is missing it. + */ + private static JsonNode normalizeBedrockMessage(JsonNode msg) { + if (!msg.has("content") || !msg.get("content").isArray()) { + return msg; + } + var mapper = BraintrustJsonMapper.get(); + ArrayNode normalizedContent = mapper.createArrayNode(); + boolean changed = false; + for (JsonNode block : msg.get("content")) { + if (block.isObject() && !block.has("type")) { + var normalized = (com.fasterxml.jackson.databind.node.ObjectNode) block.deepCopy(); + if (block.has("text")) { + normalized.put("type", "text"); + changed = true; + } else if (block.has("toolUse")) { + normalized.put("type", "tool_use"); + changed = true; + } else if (block.has("toolResult")) { + normalized.put("type", "tool_result"); + changed = true; + } else if (block.has("image")) { + normalized.put("type", "image"); + changed = true; + } + normalizedContent.add(normalized); + } else { + normalizedContent.add(block); + } + } + if (!changed) { + return msg; + } + var result = (com.fasterxml.jackson.databind.node.ObjectNode) msg.deepCopy(); + result.set("content", normalizedContent); + return result; + } + + /** Returns the Bedrock endpoint name from the last URL path segment (e.g. "converse"). */ + private static String bedrockEndpoint(List pathSegments) { + if (pathSegments.isEmpty()) { + return "unknown"; + } + return pathSegments.get(pathSegments.size() - 1); + } + private static String getSpanName(String providerName, List pathSegments) { if (pathSegments.isEmpty()) { return UNSET_LLM_SPAN_NAME; diff --git a/btx/build.gradle b/btx/build.gradle index 27de2956..150600cf 100644 --- a/btx/build.gradle +++ b/btx/build.gradle @@ -21,6 +21,7 @@ dependencies { testImplementation project(':braintrust-sdk:instrumentation:genai_1_18_0') testImplementation project(':braintrust-sdk:instrumentation:langchain_1_8_0') testImplementation project(':braintrust-sdk:instrumentation:springai_1_0_0') + testImplementation project(':braintrust-sdk:instrumentation:aws_bedrock_2_30_0') // Jackson for JSON processing testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1' @@ -31,6 +32,10 @@ dependencies { // Anthropic SDK testImplementation 'com.anthropic:anthropic-java:2.10.0' + // AWS Bedrock SDK + testImplementation 'software.amazon.awssdk:bedrockruntime:2.30.0' + testImplementation 'software.amazon.awssdk:netty-nio-client:2.30.0' + // Gemini SDK testImplementation 'org.springframework.ai:spring-ai-google-genai:1.1.0' @@ -95,6 +100,14 @@ tasks.register('fetchSpec', Exec) { } test { - dependsOn fetchSpec - systemProperty 'btx.spec.root', specOutputDir.get().asFile.absolutePath + '/test/llm_span' + // BTX_SPEC_ROOT env var lets you point at a local checkout of braintrust-spec instead of the + // fetched copy. When set, fetchSpec is skipped and the system property is set to that path. + // Example: BTX_SPEC_ROOT=/Users/you/braintrust/spec/test/llm_span ./gradlew btx:test + def specRootEnv = System.getenv('BTX_SPEC_ROOT') + if (specRootEnv) { + systemProperty 'btx.spec.root', specRootEnv + } else { + dependsOn fetchSpec + systemProperty 'btx.spec.root', specOutputDir.get().asFile.absolutePath + '/test/llm_span' + } } diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java index fa24035f..f4f18ca1 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java @@ -153,6 +153,31 @@ private static Object transformContentPart(Object part) { return part; } + // Bedrock: {type: image, image: {format: "png", source: {bytes: ""}}} + if ("image".equals(p.get("type")) && p.get("image") instanceof Map) { + // TODO: + // The Braintrust backend does not transform Bedrock image bytes into attachments, + // so we pass this shape through unchanged to match what the backend stores. + // + // Map imageMap = (Map) p.get("image"); + // if (imageMap.get("source") instanceof Map) { + // Map source = (Map) imageMap.get("source"); + // String bytes = (String) source.get("bytes"); + // if (bytes != null) { + // String format = (String) imageMap.getOrDefault("format", "png"); + // String mimeType = "image/" + format; + // Map attachment = + // toAttachment("data:" + mimeType + ";base64," + bytes); + // Map newImage = new LinkedHashMap<>(imageMap); + // newImage.put("source", attachment); + // Map newPart = new LinkedHashMap<>(p); + // newPart.put("image", newImage); + // return newPart; + // } + // } + return part; + } + if (!"image_url".equals(p.get("type"))) return part; Object imageUrlObj = p.get("image_url"); diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java index 238ea897..40dda005 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java @@ -17,8 +17,10 @@ import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseInputItem; import com.openai.models.responses.ResponseOutputItem; +import dev.braintrust.Bedrock30TestUtils; import dev.braintrust.TestHarness; import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic; +import dev.braintrust.instrumentation.awsbedrock.v2_30_0.BraintrustAWSBedrock; import dev.braintrust.instrumentation.genai.BraintrustGenAI; import dev.braintrust.instrumentation.langchain.BraintrustLangchain; import dev.braintrust.instrumentation.openai.BraintrustOpenAI; @@ -53,6 +55,16 @@ import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.function.FunctionToolCallback; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.bedrockruntime.model.ContentBlock; +import software.amazon.awssdk.services.bedrockruntime.model.ConversationRole; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler; +import software.amazon.awssdk.services.bedrockruntime.model.ImageBlock; +import software.amazon.awssdk.services.bedrockruntime.model.ImageFormat; +import software.amazon.awssdk.services.bedrockruntime.model.ImageSource; +import software.amazon.awssdk.services.bedrockruntime.model.Message; /** * Executes LLM spec tests in-process using the Braintrust Java SDK instrumentation. @@ -73,6 +85,7 @@ public class SpecExecutor { private final String openAiApiKey; private final String anthropicBaseUrl; private final String anthropicApiKey; + private final Bedrock30TestUtils bedrockUtils; private final io.opentelemetry.api.OpenTelemetry otel; public SpecExecutor(TestHarness harness) { @@ -83,6 +96,7 @@ public SpecExecutor(TestHarness harness) { this.openAiApiKey = harness.openAiApiKey(); this.anthropicBaseUrl = harness.anthropicBaseUrl(); this.anthropicApiKey = harness.anthropicApiKey(); + this.bedrockUtils = new Bedrock30TestUtils(harness); this.openAIClient = BraintrustOpenAI.wrapOpenAI( @@ -155,6 +169,10 @@ private void dispatchRequest( } else { executeAnthropicMessages(request); } + } else if ("bedrock".equals(provider) && endpoint.contains("/converse-stream")) { + executeBedrockConverseStream(request); + } else if ("bedrock".equals(provider) && endpoint.contains("/converse")) { + executeBedrockConverse(request); } else if ("google".equals(provider) && endpoint.contains(":generateContent")) { executeGeminiGenerateContent(request, endpoint); } else { @@ -615,6 +633,87 @@ private void executeAnthropicMessages(Map request) throws Except } } + // ---- AWS Bedrock ------------------------------------------------------------ + + @SuppressWarnings("unchecked") + private void executeBedrockConverse(Map request) { + String modelId = (String) request.get("modelId"); + + // Build messages from the spec YAML format: [{role, content: [{text: ...} | {image: ...}]}] + List messages = new ArrayList<>(); + for (Map msg : (List>) request.get("messages")) { + String role = (String) msg.get("role"); + List contentBlocks = new ArrayList<>(); + for (Map part : (List>) msg.get("content")) { + if (part.containsKey("text")) { + contentBlocks.add(ContentBlock.fromText((String) part.get("text"))); + } else if (part.containsKey("image")) { + contentBlocks.add( + buildBedrockImageBlock((Map) part.get("image"))); + } + } + messages.add( + Message.builder() + .role(ConversationRole.fromValue(role)) + .content(contentBlocks) + .build()); + } + + var builder = BraintrustAWSBedrock.wrap(otel, bedrockUtils.syncClientBuilder()); + try (var client = builder.build()) { + client.converse(ConverseRequest.builder().modelId(modelId).messages(messages).build()); + } + } + + @SuppressWarnings("unchecked") + private void executeBedrockConverseStream(Map request) throws Exception { + String modelId = (String) request.get("modelId"); + + List messages = new ArrayList<>(); + for (Map msg : (List>) request.get("messages")) { + String role = (String) msg.get("role"); + List contentBlocks = new ArrayList<>(); + for (Map part : (List>) msg.get("content")) { + if (part.containsKey("text")) { + contentBlocks.add(ContentBlock.fromText((String) part.get("text"))); + } + } + messages.add( + Message.builder() + .role(ConversationRole.fromValue(role)) + .content(contentBlocks) + .build()); + } + + var asyncBuilder = BraintrustAWSBedrock.wrap(otel, bedrockUtils.asyncClientBuilder()); + try (var client = asyncBuilder.build()) { + client.converseStream( + ConverseStreamRequest.builder() + .modelId(modelId) + .messages(messages) + .build(), + ConverseStreamResponseHandler.builder() + .subscriber( + ConverseStreamResponseHandler.Visitor.builder().build()) + .build()) + .get(); + } + } + + /** Builds a Bedrock {@link ContentBlock} image from the YAML {@code image:} map. */ + @SuppressWarnings("unchecked") + private static ContentBlock buildBedrockImageBlock(Map imageMap) { + String format = (String) imageMap.getOrDefault("format", "png"); + Map sourceMap = (Map) imageMap.get("source"); + String base64 = (String) sourceMap.get("bytes"); + byte[] imageBytes = java.util.Base64.getDecoder().decode(base64); + return ContentBlock.fromImage( + ImageBlock.builder() + .format(ImageFormat.fromValue(format)) + .source(ImageSource.fromBytes(SdkBytes.fromByteArray(imageBytes))) + .build()); + } + // ---- Google Gemini ---------------------------------------------------------- @SuppressWarnings("unchecked") diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java index 711cdee3..1eef02ba 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java @@ -33,9 +33,16 @@ public class SpecLoader { /** - * The spec root directory. When the {@code btx.spec.root} system property is set (by the {@code - * fetchSpec} Gradle task), that path is used. Otherwise falls back to the legacy in-tree - * location for local development. + * The spec root directory. Resolved in priority order: + * + *

    + *
  1. The {@code btx.spec.root} system property — set by the Gradle {@code test} task to the + * output of {@code fetchSpec}, or to {@code $BTX_SPEC_ROOT} when that env var is set. + *
  2. The fallback {@code btx/spec/llm_span} in-tree path for ad-hoc local runs. + *
+ * + *

To use a local checkout of braintrust-spec: {@code + * BTX_SPEC_ROOT=/path/to/spec/test/llm_span ./gradlew btx:test} */ private static final String SPEC_ROOT = System.getProperty("btx.spec.root", "btx/spec/llm_span"); @@ -48,7 +55,8 @@ public class SpecLoader { static final Map> CLIENTS_BY_PROVIDER = Map.of( "openai", List.of("openai", "langchain-openai", "springai-openai"), - "anthropic", List.of("anthropic", "springai-anthropic")); + "anthropic", List.of("anthropic", "springai-anthropic"), + "bedrock", List.of("bedrock")); /** * Returns the clients to test for the given provider. Defaults to {@code [providerName]} if the diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy index a282f9da..b4f78eb2 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy @@ -40,6 +40,16 @@ class MuzzleDirective { */ List ignoredInstrumentation = [] + /** + * Explicit list of versions to check instead of resolving a range from Maven Central. + * When set, {@code versions} is ignored and no network fetch is performed. + */ + List pinnedVersions = [] + + void pinVersions(String... versions) { + pinnedVersions.addAll(versions) + } + void skipVersions(String... versions) { skipVersions.addAll(versions) } @@ -54,6 +64,7 @@ class MuzzleDirective { @Override String toString() { - "${assertPass ? 'pass' : 'fail'} { ${group}:${module}:${versions} }" + def versionStr = pinnedVersions ? "pinned[${pinnedVersions.join(', ')}]" : versions + "${assertPass ? 'pass' : 'fail'} { ${group}:${module}:${versionStr} }" } } diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleExtension.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleExtension.groovy index f561f541..5fcdd2c4 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleExtension.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleExtension.groovy @@ -38,8 +38,8 @@ class MuzzleExtension { if (!directive.module) { throw new IllegalArgumentException("muzzle directive requires 'module'") } - if (!directive.versions) { - throw new IllegalArgumentException("muzzle directive requires 'versions'") + if (!directive.pinnedVersions && !directive.versions) { + throw new IllegalArgumentException("muzzle directive requires either 'versions' or 'pinVersions'") } } } diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy index 2d667c30..6dc68a0e 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy @@ -47,16 +47,21 @@ class MuzzleTask extends DefaultTask { logger.lifecycle("[muzzle] Checking: ${directive}") List versions - try { - versions = MavenVersions.resolve( - directive.group, directive.module, directive.versions, directive.skipVersions) - } catch (Exception e) { - throw new GradleException("[muzzle] Failed to resolve versions for ${directive}: ${e.message}", e) - } + if (directive.pinnedVersions) { + versions = directive.pinnedVersions + logger.lifecycle("[muzzle] Using ${versions.size()} pinned version(s)") + } else { + try { + versions = MavenVersions.resolve( + directive.group, directive.module, directive.versions, directive.skipVersions) + } catch (Exception e) { + throw new GradleException("[muzzle] Failed to resolve versions for ${directive}: ${e.message}", e) + } - if (versions.isEmpty()) { - logger.warn("[muzzle] No versions found for ${directive.group}:${directive.module} in range ${directive.versions}") - continue + if (versions.isEmpty()) { + logger.warn("[muzzle] No versions found for ${directive.group}:${directive.module} in range ${directive.versions}") + continue + } } logger.lifecycle("[muzzle] Found ${versions.size()} version(s) to check") diff --git a/examples/build.gradle b/examples/build.gradle index 8bbe1a4d..6a231b18 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -25,6 +25,7 @@ dependencies { implementation project(':braintrust-sdk:instrumentation:genai_1_18_0') implementation project(':braintrust-sdk:instrumentation:langchain_1_8_0') implementation project(':braintrust-sdk:instrumentation:springai_1_0_0') + implementation project(':braintrust-sdk:instrumentation:aws_bedrock_2_30_0') runtimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}" // To run otel examples implementation "io.opentelemetry:opentelemetry-exporter-otlp:${otelVersion}" @@ -36,6 +37,7 @@ dependencies { implementation 'com.google.genai:google-genai:1.20.0' // spring ai examples implementation 'org.springframework.ai:spring-ai-anthropic:1.1.0' + implementation 'org.springframework.ai:spring-ai-bedrock-converse:1.1.0' implementation 'org.springframework.ai:spring-ai-google-genai:1.1.0' implementation 'org.springframework.ai:spring-ai-openai:1.1.0' // spring-ai-openai requires spring-webflux (WebClient) at runtime @@ -47,6 +49,7 @@ dependencies { // to run langchain4j examples implementation 'dev.langchain4j:langchain4j:1.9.1' implementation 'dev.langchain4j:langchain4j-open-ai:1.9.1' + } application { diff --git a/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java b/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java index cbe6d7b8..e8a659e0 100644 --- a/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java +++ b/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java @@ -3,15 +3,20 @@ import com.google.genai.Client; import dev.braintrust.Braintrust; import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.instrumentation.awsbedrock.v2_30_0.BraintrustAWSBedrock; import dev.braintrust.instrumentation.genai.BraintrustGenAI; import dev.braintrust.instrumentation.springai.v1_0_0.BraintrustSpringAI; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; +import java.util.ArrayList; +import java.util.List; import org.springframework.ai.anthropic.AnthropicChatModel; import org.springframework.ai.anthropic.AnthropicChatOptions; import org.springframework.ai.anthropic.api.AnthropicApi; +import org.springframework.ai.bedrock.converse.BedrockChatOptions; +import org.springframework.ai.bedrock.converse.BedrockProxyChatModel; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.google.genai.GoogleGenAiChatModel; @@ -19,13 +24,14 @@ import org.springframework.ai.openai.OpenAiChatModel; import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.ai.openai.api.OpenAiApi; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.http.client.HttpClientAutoConfiguration; import org.springframework.boot.autoconfigure.web.client.RestClientAutoConfiguration; import org.springframework.context.annotation.Bean; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient; /** Spring Boot application demonstrating Braintrust + Spring AI integration */ @SpringBootApplication( @@ -41,29 +47,23 @@ public static void main(String[] args) { } @Bean - public CommandLineRunner run( - @Qualifier("openAIChatModel") ChatModel openAIChatModel, - @Qualifier("anthropicChatModel") ChatModel anthropicChatModel, - @Qualifier("googleChatModel") ChatModel googleChatModel, - Tracer tracer, - Braintrust braintrust) { + public CommandLineRunner run(List chatModels, Tracer tracer, Braintrust braintrust) { return args -> { Span rootSpan = tracer.spanBuilder("spring-ai-example").startSpan(); try (Scope scope = rootSpan.makeCurrent()) { System.out.println("\n=== Running Spring Boot Example ===\n"); - // Make a simple chat call var prompt = new Prompt("what's the name of the most popular java DI framework?"); - var oaiResponse = openAIChatModel.call(prompt); - var anthropicResponse = anthropicChatModel.call(prompt); - var googleResponse = googleChatModel.call(prompt); - - System.out.println( - "~~~ SPRING AI CHAT RESPONSES: \noat: %s\nanthropic: %s\ngoogle: %s\n" - .formatted( - oaiResponse.getResult().getOutput().getText(), - anthropicResponse.getResult().getOutput().getText(), - googleResponse.getResult().getOutput().getText())); + + System.out.println("~~~ SPRING AI CHAT RESPONSES:"); + for (var model : chatModels) { + var response = model.call(prompt); + System.out.println( + model.getClass().getSimpleName() + + ": " + + response.getResult().getOutput().getText()); + } + System.out.println(); } finally { rootSpan.end(); } @@ -80,6 +80,90 @@ public CommandLineRunner run( }; } + @Bean + public List chatModels(OpenTelemetry openTelemetry) { + var models = new ArrayList(); + + if (System.getenv("OPENAI_API_KEY") != null) { + models.add( + BraintrustSpringAI.wrap( + openTelemetry, + OpenAiChatModel.builder() + .openAiApi( + OpenAiApi.builder() + .apiKey(System.getenv("OPENAI_API_KEY")) + .build()) + .defaultOptions( + OpenAiChatOptions.builder() + .model("gpt-4o-mini") + .temperature(0.0) + .maxTokens(50) + .build())) + .build()); + } + + if (System.getenv("ANTHROPIC_API_KEY") != null) { + models.add( + BraintrustSpringAI.wrap( + openTelemetry, + AnthropicChatModel.builder() + .anthropicApi( + AnthropicApi.builder() + .apiKey( + System.getenv( + "ANTHROPIC_API_KEY")) + .build()) + .defaultOptions( + AnthropicChatOptions.builder() + .model("claude-3-haiku-20240307") + .temperature(0.0) + .maxTokens(50) + .build())) + .build()); + } + + if (System.getenv("GOOGLE_API_KEY") != null || System.getenv("GEMINI_API_KEY") != null) { + models.add( + GoogleGenAiChatModel.builder() + .genAiClient(BraintrustGenAI.wrap(openTelemetry, new Client.Builder())) + .defaultOptions( + GoogleGenAiChatOptions.builder() + .model("gemini-2.0-flash-lite") + .temperature(0.0) + .maxOutputTokens(50) + .build()) + .build()); + } + + if (System.getenv("AWS_ACCESS_KEY_ID") != null + && System.getenv("AWS_SECRET_ACCESS_KEY") != null) { + var bedrockClient = + BraintrustAWSBedrock.wrap(openTelemetry, BedrockRuntimeClient.builder()) + .build(); + models.add( + BedrockProxyChatModel.builder() + .bedrockRuntimeClient(bedrockClient) + .region(Region.US_EAST_1) + .defaultOptions( + BedrockChatOptions.builder() + // .model("us.anthropic.claude-haiku-4-5-20251001-v1:0") + .model("us.amazon.nova-lite-v1:0") + .temperature(0.0) + .maxTokens(50) + .build()) + .build()); + } + + if (models.isEmpty()) { + System.err.println( + "\nWARNING: No API keys found. Set at least one of: OPENAI_API_KEY," + + " ANTHROPIC_API_KEY, GOOGLE_API_KEY/GEMINI_API_KEY, or" + + " AWS_ACCESS_KEY_ID+AWS_SECRET_ACCESS_KEY\n"); + } + + return models; + } + @Bean public Braintrust braintrust() { return Braintrust.get(BraintrustConfig.fromEnvironment()); @@ -94,75 +178,4 @@ public OpenTelemetry openTelemetry(Braintrust braintrust) { public Tracer tracer(OpenTelemetry openTelemetry) { return openTelemetry.getTracer("spring-ai-instrumentation"); } - - @Bean - public ChatModel openAIChatModel(OpenTelemetry openTelemetry) { - if (null == System.getenv("OPENAI_API_KEY")) { - System.err.println( - "\n" - + "WARNING: OPENAI_API_KEY not found. This example will likely" - + " fail.\n" - + "Set it with: export OPENAI_API_KEY='your-key'\n"); - } - return BraintrustSpringAI.wrap( - openTelemetry, - OpenAiChatModel.builder() - .openAiApi( - OpenAiApi.builder() - .apiKey(System.getenv("OPENAI_API_KEY")) - .build()) - .defaultOptions( - OpenAiChatOptions.builder() - .model("gpt-4o-mini") - .temperature(0.0) - .maxTokens(50) - .build())) - .build(); - } - - @Bean - public ChatModel anthropicChatModel(OpenTelemetry openTelemetry) { - if (null == System.getenv("ANTHROPIC_API_KEY")) { - System.err.println( - "\n" - + "WARNING: ANTHROPIC_API_KEY not found. This example will" - + " likely fail.\n" - + "Set it with: export ANTHROPIC_API_KEY='your-key'\n"); - } - return BraintrustSpringAI.wrap( - openTelemetry, - AnthropicChatModel.builder() - .anthropicApi( - AnthropicApi.builder() - .apiKey(System.getenv("ANTHROPIC_API_KEY")) - .build()) - .defaultOptions( - AnthropicChatOptions.builder() - .model("claude-3-haiku-20240307") - .temperature(0.0) - .maxTokens(50) - .build())) - .build(); - } - - @Bean - public ChatModel googleChatModel(OpenTelemetry openTelemetry) { - if (null == System.getenv("GOOGLE_API_KEY") && null == System.getenv("GEMINI_API_KEY")) { - System.err.println( - "\n" - + "WARNING: Neither GOOGLE_API_KEY nor GEMINI_API_KEY found. This" - + " example will likely fail.\n" - + "Set either: export GOOGLE_API_KEY='your-key' (recommended) or" - + " export GEMINI_API_KEY='your-key'\n"); - } - return GoogleGenAiChatModel.builder() - .genAiClient(BraintrustGenAI.wrap(openTelemetry, new Client.Builder())) - .defaultOptions( - GoogleGenAiChatOptions.builder() - .model("gemini-2.0-flash-lite") - .temperature(0.0) - .maxOutputTokens(50) - .build()) - .build(); - } } diff --git a/gradle.properties b/gradle.properties index 9ce137d5..21de6bcd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,8 +8,7 @@ org.gradle.daemon=true org.gradle.warning.mode=summary # braintrust-spec git ref (SHA or tag) used by btx tests -# TODO: update this to point at a tag once specs publish their first release -braintrustSpecRef=v0.0.1 +braintrustSpecRef=v0.0.2 # Let Gradle locate local JDKs and download one if needed org.gradle.java.installations.auto-detect=true diff --git a/settings.gradle b/settings.gradle index 7aeeb942..67e00999 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,6 +16,7 @@ include 'braintrust-sdk:instrumentation:anthropic_2_2_0' include 'braintrust-sdk:instrumentation:genai_1_18_0' include 'braintrust-sdk:instrumentation:langchain_1_8_0' include 'braintrust-sdk:instrumentation:springai_1_0_0' +include 'braintrust-sdk:instrumentation:aws_bedrock_2_30_0' include 'braintrust-java-agent:smoke-test:test-instrumentation' include 'braintrust-java-agent:smoke-test:dd-agent' include 'braintrust-java-agent:smoke-test:otel-agent' diff --git a/test-harness/build.gradle b/test-harness/build.gradle index 67068d96..9cbf0a74 100644 --- a/test-harness/build.gradle +++ b/test-harness/build.gradle @@ -41,4 +41,9 @@ dependencies { testFixturesImplementation 'com.google.code.findbugs:jsr305:3.0.2' testFixturesImplementation "org.slf4j:slf4j-api:${slf4jVersion}" testFixturesImplementation "org.slf4j:slf4j-simple:${slf4jVersion}" + + // Bedrock client builders used by Bedrock30TestUtils — compileOnly so consumers + // that don't use Bedrock aren't forced to pull in the AWS SDK. + testFixturesCompileOnly 'software.amazon.awssdk:bedrockruntime:2.30.0' + testFixturesCompileOnly 'software.amazon.awssdk:netty-nio-client:2.30.0' } diff --git a/test-harness/src/testFixtures/java/dev/braintrust/Bedrock30TestUtils.java b/test-harness/src/testFixtures/java/dev/braintrust/Bedrock30TestUtils.java new file mode 100644 index 00000000..077a3617 --- /dev/null +++ b/test-harness/src/testFixtures/java/dev/braintrust/Bedrock30TestUtils.java @@ -0,0 +1,104 @@ +package dev.braintrust; + +import java.net.URI; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.Protocol; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClient; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClientBuilder; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClientBuilder; + +/** + * Shared test utilities for constructing AWS Bedrock Runtime client builders wired to the test + * harness (WireMock endpoint override, SigV4 host rewrite, replay credentials). + */ +public class Bedrock30TestUtils { + + public static final String BEDROCK_REGION = "us-east-1"; + public static final String BEDROCK_REAL_HOST = + "bedrock-runtime." + BEDROCK_REGION + ".amazonaws.com"; + + private static final StaticCredentialsProvider REPLAY_CREDS = + StaticCredentialsProvider.create(AwsBasicCredentials.create("fake-key", "fake-secret")); + + private final TestHarness testHarness; + + public Bedrock30TestUtils(TestHarness testHarness) { + this.testHarness = testHarness; + } + + /** + * Returns a {@link BedrockRuntimeClientBuilder} pointed at the test harness WireMock endpoint, + * with the SigV4 host-rewrite interceptor applied and fake credentials in replay mode. + */ + public BedrockRuntimeClientBuilder syncClientBuilder() { + var builder = + BedrockRuntimeClient.builder() + .overrideConfiguration( + ClientOverrideConfiguration.builder() + .addExecutionInterceptor( + new HostRewriteInterceptor(BEDROCK_REAL_HOST)) + .build()) + .region(Region.of(BEDROCK_REGION)) + .endpointOverride(URI.create(testHarness.bedrockBaseUrl(BEDROCK_REGION))); + + if (TestHarness.getVcrMode() == VCR.VcrMode.REPLAY) { + builder.credentialsProvider(REPLAY_CREDS); + } + + return builder; + } + + /** + * Returns a {@link BedrockRuntimeAsyncClientBuilder} pointed at the test harness WireMock + * endpoint, with the SigV4 host-rewrite interceptor, HTTP/1.1 forced (so WireMock can + * proxy/replay the event-stream), and fake credentials in replay mode. + */ + public BedrockRuntimeAsyncClientBuilder asyncClientBuilder() { + var builder = + BedrockRuntimeAsyncClient.builder() + .overrideConfiguration( + ClientOverrideConfiguration.builder() + .addExecutionInterceptor( + new HostRewriteInterceptor(BEDROCK_REAL_HOST)) + .build()) + .region(Region.of(BEDROCK_REGION)) + .endpointOverride(URI.create(testHarness.bedrockBaseUrl(BEDROCK_REGION))) + // Force HTTP/1.1 so WireMock can proxy/replay the event-stream + .httpClientBuilder( + NettyNioAsyncHttpClient.builder().protocol(Protocol.HTTP1_1)); + + if (TestHarness.getVcrMode() == VCR.VcrMode.REPLAY) { + builder.credentialsProvider(REPLAY_CREDS); + } + + return builder; + } + + /** + * Rewrites the {@code Host} header to the real AWS hostname before SigV4 signing, so that + * signatures are valid even when the request is sent to the local WireMock proxy via {@code + * endpointOverride}. + */ + public static class HostRewriteInterceptor implements ExecutionInterceptor { + private final String realHost; + + public HostRewriteInterceptor(String realHost) { + this.realHost = realHost; + } + + @Override + public SdkHttpRequest modifyHttpRequest( + software.amazon.awssdk.core.interceptor.Context.ModifyHttpRequest context, + ExecutionAttributes executionAttributes) { + return context.httpRequest().toBuilder().putHeader("Host", realHost).build(); + } + } +} diff --git a/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java b/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java index f45bd8ba..5540c06f 100644 --- a/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java +++ b/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java @@ -35,7 +35,10 @@ public class TestHarness { getEnv("OPENAI_API_KEY", ""), getEnv("ANTHROPIC_API_KEY", ""), getEnv("GOOGLE_API_KEY", getEnv("GEMINI_API_KEY", "")), - getEnv("BRAINTRUST_API_KEY", "")); + getEnv("BRAINTRUST_API_KEY", ""), + getEnv("AWS_ACCESS_KEY_ID", ""), + getEnv("AWS_SECRET_ACCESS_KEY", ""), + getEnv("AWS_SESSION_TOKEN", "")); vcr = new VCR( @@ -43,7 +46,8 @@ public class TestHarness { "https://api.openai.com/v1", "openai", "https://api.anthropic.com", "anthropic", "https://generativelanguage.googleapis.com", "google", - "https://api.braintrust.dev", "braintrust"), + "https://api.braintrust.dev", "braintrust", + "https://bedrock-runtime.us-east-1.amazonaws.com", "bedrock"), apiKeysToNeverRecord); vcr.start(); UnitTestShutdownHook.addShutdownHook(1, vcr::stop); @@ -157,6 +161,17 @@ public String googleApiKey() { return getEnv("GOOGLE_API_KEY", getEnv("GEMINI_API_KEY", "test-key")); } + /** + * Returns the VCR proxy URL for the Bedrock Runtime endpoint in the given region. The region + * must have been registered in the VCR target map at init time. + */ + public String bedrockBaseUrl(String region) { + if (!"us-east-1".equals(region)) { + throw new RuntimeException("unsupported region: " + region); + } + return vcr.getUrlForTargetBase("https://bedrock-runtime." + region + ".amazonaws.com"); + } + public String braintrustApiBaseUrl() { return braintrust.config().apiUrl(); } diff --git a/test-harness/src/testFixtures/java/dev/braintrust/VCR.java b/test-harness/src/testFixtures/java/dev/braintrust/VCR.java index c5219ffe..26d95d4c 100644 --- a/test-harness/src/testFixtures/java/dev/braintrust/VCR.java +++ b/test-harness/src/testFixtures/java/dev/braintrust/VCR.java @@ -258,10 +258,13 @@ private void createProgrammaticStubFromMapping( // Extract request body pattern for matching JsonNode bodyPatterns = mapping.at("/request/bodyPatterns"); - // Check if this is an SSE response + // Check if this is an SSE response or binary AWS event-stream response JsonNode contentType = mapping.at("/response/headers/Content-Type"); boolean isSse = contentType.isTextual() && contentType.asText().contains("text/event-stream"); + boolean isEventStream = + contentType.isTextual() + && contentType.asText().contains("application/vnd.amazon.eventstream"); // Check if this is a function invoke request with dynamic OTEL parent info // Only handle invoke requests that have parent.row_ids (which contains dynamic trace IDs) @@ -277,12 +280,16 @@ private void createProgrammaticStubFromMapping( } } - if (!isSse && !isFunctionInvokeWithParent) { + if (!isSse && !isEventStream && !isFunctionInvokeWithParent) { return; // Let WireMock handle other responses normally } if (isSse) { log.info("Creating programmatic stub for SSE response: " + mappingPath.getFileName()); + } else if (isEventStream) { + log.info( + "Creating programmatic stub for binary event-stream response: " + + mappingPath.getFileName()); } else { log.info( "Creating programmatic stub for function invoke with parent: " @@ -294,17 +301,6 @@ private void createProgrammaticStubFromMapping( String responseContentType = contentType.isTextual() ? contentType.asText() : "application/json"; - String body; - if (mapping.at("/response/body").isTextual()) { - body = mapping.at("/response/body").asText(); - } else if (mapping.at("/response/bodyFileName").isTextual()) { - String bodyFileName = mapping.at("/response/bodyFileName").asText(); - Path bodyPath = Paths.get(cassettesRoot, mappingsDir, "__files", bodyFileName); - body = Files.readString(bodyPath); - } else { - return; - } - // Create programmatic stub com.github.tomakehurst.wiremock.client.MappingBuilder stub = com.github.tomakehurst.wiremock.client.WireMock.request( @@ -330,10 +326,29 @@ private void createProgrammaticStubFromMapping( com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder response = com.github.tomakehurst.wiremock.client.WireMock.aResponse() .withStatus(status) - .withHeader("Content-Type", responseContentType) - .withBody(body); + .withHeader("Content-Type", responseContentType); + + // Binary event-stream bodies must be served as raw bytes to avoid UTF-8 corruption + if (isEventStream) { + if (mapping.at("/response/bodyFileName").isTextual()) { + String bodyFileName = mapping.at("/response/bodyFileName").asText(); + Path bodyPath = Paths.get(cassettesRoot, mappingsDir, "__files", bodyFileName); + response.withBody(Files.readAllBytes(bodyPath)); + } else { + return; + } + } else { + if (mapping.at("/response/body").isTextual()) { + response.withBody(mapping.at("/response/body").asText()); + } else if (mapping.at("/response/bodyFileName").isTextual()) { + String bodyFileName = mapping.at("/response/bodyFileName").asText(); + Path bodyPath = Paths.get(cassettesRoot, mappingsDir, "__files", bodyFileName); + response.withBody(Files.readString(bodyPath)); + } else { + return; + } + } - // Use instance method wireMock.stubFor(stub.willReturn(response)); } diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json new file mode 100644 index 00000000..c4425296 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":641},"output":{"message":{"content":[{"text":"Sorry, I can't describe an image. I can only provide information about the color. However, if you want to know about the color of the image, I can provide information about it. The image is in a red color."}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":536,"outputTokens":49,"serverToolUsage":{},"totalTokens":585}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json new file mode 100644 index 00000000..14bc9a83 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":888},"output":{"message":{"content":[{"text":"The capital of France is Paris. Paris is not only the capital but also the largest city in France. It is situated in the northern central part of the country, along the Seine River. Paris is renowned for its rich history, culture, and landmarks. Some of its most famous attractions include the Eiffel Tower, the Louvre Museum, Notre-Dame Cathedral, and the Champs-Élysées. It is a major global city and a hub for art, fashion, gastronomy, and diplomacy. Paris is divided into 20 arrondissements (municipalities), each with its own unique character and attractions. The city is also known for its significant contributions to various fields such as philosophy, science, and literature."}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":7,"outputTokens":144,"serverToolUsage":{},"totalTokens":151}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json new file mode 100644 index 00000000..56d5b57c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":289},"output":{"message":{"content":[{"text":"Paris"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":12,"outputTokens":2,"serverToolUsage":{},"totalTokens":14}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.txt b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.txt new file mode 100644 index 00000000..f0c60b84 Binary files /dev/null and b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.txt differ diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json new file mode 100644 index 00000000..ad7ae1fe --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json @@ -0,0 +1 @@ +{"metrics":{"latencyMs":254},"output":{"message":{"content":[{"text":"Paris."}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":19,"outputTokens":5,"serverToolUsage":{},"totalTokens":24}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.txt b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.txt new file mode 100644 index 00000000..121ff659 Binary files /dev/null and b/test-harness/src/testFixtures/resources/cassettes/bedrock/__files/model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.txt differ diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json new file mode 100644 index 00000000..fb104517 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json @@ -0,0 +1,30 @@ +{ + "id" : "400d471f-3f4d-4d3e-9571-cb3f015011c5", + "name" : "model_us.amazon.nova-lite-v10_converse", + "request" : { + "url" : "/model/us.amazon.nova-lite-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What color is this image?\"},{\"image\":{\"format\":\"png\",\"source\":{\"bytes\":\"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==\"}}}]}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.amazon.nova-lite-v10_converse-400d471f-3f4d-4d3e-9571-cb3f015011c5.json", + "headers" : { + "x-amzn-RequestId" : "4d00f377-e51b-4659-b097-044878f09cc4", + "Date" : "Sun, 19 Apr 2026 23:33:09 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "400d471f-3f4d-4d3e-9571-cb3f015011c5", + "persistent" : true, + "insertionIndex" : 6 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json new file mode 100644 index 00000000..2656f20e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json @@ -0,0 +1,30 @@ +{ + "id" : "57a96b00-9f78-4336-89e1-75f8e511afd2", + "name" : "model_us.amazon.nova-lite-v10_converse", + "request" : { + "url" : "/model/us.amazon.nova-lite-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What is the capital of France?\"}]}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.amazon.nova-lite-v10_converse-57a96b00-9f78-4336-89e1-75f8e511afd2.json", + "headers" : { + "x-amzn-RequestId" : "f08deaf6-8368-4126-bcdf-8b659e1cf98e", + "Date" : "Sun, 19 Apr 2026 23:33:09 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "57a96b00-9f78-4336-89e1-75f8e511afd2", + "persistent" : true, + "insertionIndex" : 5 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json new file mode 100644 index 00000000..c8522735 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json @@ -0,0 +1,30 @@ +{ + "id" : "d2a2ec99-c9bf-48e6-b0c6-eb53a928e698", + "name" : "model_us.amazon.nova-lite-v10_converse", + "request" : { + "url" : "/model/us.amazon.nova-lite-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What is the capital of France? Reply in one word.\"}]}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.amazon.nova-lite-v10_converse-d2a2ec99-c9bf-48e6-b0c6-eb53a928e698.json", + "headers" : { + "x-amzn-RequestId" : "f62b4526-a86c-4273-a4a7-f8de296a8974", + "Date" : "Sun, 19 Apr 2026 23:32:58 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "d2a2ec99-c9bf-48e6-b0c6-eb53a928e698", + "persistent" : true, + "insertionIndex" : 1 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.json new file mode 100644 index 00000000..1390180b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.json @@ -0,0 +1,30 @@ +{ + "id" : "df6a23a1-b3d8-403e-a499-bc5d257af868", + "name" : "model_us.amazon.nova-lite-v10_converse-stream", + "request" : { + "url" : "/model/us.amazon.nova-lite-v1%3A0/converse-stream", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"count to 10 slowly\"}]}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.amazon.nova-lite-v10_converse-stream-df6a23a1-b3d8-403e-a499-bc5d257af868.txt", + "headers" : { + "x-amzn-RequestId" : "3f9aae37-90f0-470c-8174-bb9ab43b9ffb", + "Date" : "Sun, 19 Apr 2026 23:33:10 GMT", + "Content-Type" : "application/vnd.amazon.eventstream" + } + }, + "uuid" : "df6a23a1-b3d8-403e-a499-bc5d257af868", + "persistent" : true, + "insertionIndex" : 4 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json new file mode 100644 index 00000000..b0d2ff0d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json @@ -0,0 +1,30 @@ +{ + "id" : "bfd89f01-c330-4c6d-b9c0-05ca56ed005f", + "name" : "model_us.anthropic.claude-3-haiku-20240307-v10_converse", + "request" : { + "url" : "/model/us.anthropic.claude-3-haiku-20240307-v1%3A0/converse", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What is the capital of France? Reply in one word.\"}]}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.anthropic.claude-3-haiku-20240307-v10_converse-bfd89f01-c330-4c6d-b9c0-05ca56ed005f.json", + "headers" : { + "x-amzn-RequestId" : "d35171f6-3a12-4cde-afd1-1ee875aa9724", + "Date" : "Sun, 19 Apr 2026 23:32:57 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "bfd89f01-c330-4c6d-b9c0-05ca56ed005f", + "persistent" : true, + "insertionIndex" : 2 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.json b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.json new file mode 100644 index 00000000..1b4d4c43 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/bedrock/mappings/model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.json @@ -0,0 +1,30 @@ +{ + "id" : "c43d383c-79df-4107-996c-7c63b26994fd", + "name" : "model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream", + "request" : { + "url" : "/model/us.anthropic.claude-3-haiku-20240307-v1%3A0/converse-stream", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"What is the capital of France? Reply in one word.\"}]}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "model_us.anthropic.claude-3-haiku-20240307-v10_converse-stream-c43d383c-79df-4107-996c-7c63b26994fd.txt", + "headers" : { + "x-amzn-RequestId" : "8befd5fa-f1c6-4407-ace6-902d2fa03a37", + "Date" : "Sun, 19 Apr 2026 23:32:53 GMT", + "Content-Type" : "application/vnd.amazon.eventstream" + } + }, + "uuid" : "c43d383c-79df-4107-996c-7c63b26994fd", + "persistent" : true, + "insertionIndex" : 3 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json new file mode 100644 index 00000000..957de69a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07630617538759426048","_xact_id":"1000197026236401311","audit_data":[{"_xact_id":"1000197026236401311","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-04-19T23:33:08.389Z","error":null,"expected":null,"facets":null,"id":"c1215a934be7d585","input":[{"content":[{"text":"What color is this image?","type":"text"},{"image":{"format":"png","source":{"bytes":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},"type":"image"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","endpoint":"converse","model":"us.amazon.nova-lite-v1:0","provider":"bedrock","request_base_uri":"http://localhost","request_method":"POST","request_path":"model/us.amazon.nova-lite-v1%3A0/converse"},"metrics":{"completion_tokens":49,"end":1776641589.5266142,"prompt_tokens":536,"start":1776641588.3893957,"tokens":585},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"content":[{"text":"Sorry, I can't describe an image. I can only provide information about the color. However, if you want to know about the color of the image, I can provide information about it. The image is in a red color.","type":"text"}],"role":"assistant"}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"8b6742b1702306836d0410b9a48a7559","scores":null,"span_attributes":{"name":"bedrock.converse","type":"llm"},"span_id":"c1215a934be7d585","span_parents":["06f8d5792201523a"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197026238071131","read_bytes":0,"actual_xact_id":"1000197026238071131"},"freshness_state":{"last_processed_xact_id":"1000197026238071131","last_considered_xact_id":"1000197026238071131"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json new file mode 100644 index 00000000..884a877a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07630617538759426050","_xact_id":"1000197026236401311","audit_data":[{"_xact_id":"1000197026236401311","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-04-19T23:33:08.389Z","error":null,"expected":null,"facets":null,"id":"718e2f54dad1ee66","input":[{"content":[{"text":"count to 10 slowly","type":"text"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","endpoint":"converse-stream","model":"us.amazon.nova-lite-v1:0","provider":"bedrock","request_base_uri":"http://localhost","request_method":"POST","request_path":"model/us.amazon.nova-lite-v1%3A0/converse-stream"},"metrics":{"completion_tokens":71,"end":1776641590.6287882,"prompt_tokens":6,"start":1776641588.3893952,"time_to_first_token":0.002601833,"tokens":77},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"content":[{"text":"Sure, I'll count to 10 slowly for you:\n\n1. One\n2. Two\n3. Three\n4. Four\n5. Five\n6. Six\n7. Seven\n8. Eight\n9. Nine\n10. Ten\n\nThere you go! If you need anything else, feel free to ask.","type":"text"}],"role":"assistant"}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"e4243df27a38be41de316d64dea6519a","scores":null,"span_attributes":{"name":"bedrock.converse-stream","type":"llm"},"span_id":"718e2f54dad1ee66","span_parents":["2357c16b6388e8e3"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197026238071131","read_bytes":0,"actual_xact_id":"1000197026238071131"},"freshness_state":{"last_processed_xact_id":"1000197026238071131","last_considered_xact_id":"1000197026238071131"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json new file mode 100644 index 00000000..64338849 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json @@ -0,0 +1 @@ +{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":null,"read_bytes":0,"actual_xact_id":null},"freshness_state":{"last_processed_xact_id":"1000197026235530733","last_considered_xact_id":null}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json new file mode 100644 index 00000000..2b97130e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07630617538759426049","_xact_id":"1000197026236401311","audit_data":[{"_xact_id":"1000197026236401311","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-04-19T23:33:08.389Z","error":null,"expected":null,"facets":null,"id":"6608c80a21becae4","input":[{"content":[{"text":"What is the capital of France?","type":"text"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","endpoint":"converse","model":"us.amazon.nova-lite-v1:0","provider":"bedrock","request_base_uri":"http://localhost","request_method":"POST","request_path":"model/us.amazon.nova-lite-v1%3A0/converse"},"metrics":{"completion_tokens":144,"end":1776641589.7047968,"prompt_tokens":7,"start":1776641588.389398,"tokens":151},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"content":[{"text":"The capital of France is Paris. Paris is not only the capital but also the largest city in France. It is situated in the northern central part of the country, along the Seine River. Paris is renowned for its rich history, culture, and landmarks. Some of its most famous attractions include the Eiffel Tower, the Louvre Museum, Notre-Dame Cathedral, and the Champs-Élysées. It is a major global city and a hub for art, fashion, gastronomy, and diplomacy. Paris is divided into 20 arrondissements (municipalities), each with its own unique character and attractions. The city is also known for its significant contributions to various fields such as philosophy, science, and literature.","type":"text"}],"role":"assistant"}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"46647b58b1eb1908a50cfc1db2d84f37","scores":null,"span_attributes":{"name":"bedrock.converse","type":"llm"},"span_id":"6608c80a21becae4","span_parents":["5f7d3eae6521275f"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_async_scoring_state":{},"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"required":["id"],"additionalProperties":false,"properties":{"confidence":{"description":"Optional confidence score for the classification","type":["number","null"]},"id":{"description":"Stable classification identifier","type":"string"},"label":{"description":"Original label of the classification item, which is useful for search and indexing purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The type of global function. Defaults to 'scorer'.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The timestamp the project logs event was created","type":"string","format":"date-time"},"error":{"description":"The error that occurred, if any."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"facets":{"anyOf":[{"additionalProperties":{},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you","type":"string"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"is_root":{"description":"Whether this span is a root span","type":["boolean","null"]},"log_id":{"description":"A literal 'g' which identifies the log as a project log","const":"g","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This metric is deprecated"},"caller_functionname":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"org_id":{"description":"Unique id for the organization that the project belongs under","type":"string","format":"uuid"},"origin":{"anyOf":[{"description":"Reference to the original object and event this was copied from.","required":["object_type","object_id","id"],"properties":{"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID of the original event.","type":"string"},"object_id":{"description":"ID of the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"type":"object"},{"type":"null"}]},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"project_id":{"description":"Unique identifier for the project","type":"string","format":"uuid"},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"description":"Human-identifying attributes of the span, such as name, type, etc.","additionalProperties":{},"properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197026238071131","read_bytes":0,"actual_xact_id":"1000197026238071131"},"freshness_state":{"last_processed_xact_id":"1000197026238071131","last_considered_xact_id":"1000197026238071131"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json new file mode 100644 index 00000000..89ea5c34 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json @@ -0,0 +1,44 @@ +{ + "id" : "1a27eb77-e3d5-4987-ae9f-4c31512eb77e", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"},\"op\":\"ne\"},\"left\":{\"right\":{\"value\":\"8b6742b1702306836d0410b9a48a7559\",\"op\":\"literal\"},\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"},\"op\":\"eq\"},\"op\":\"and\"},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-1a27eb77-e3d5-4987-ae9f-4c31512eb77e.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFztuFYrIAMEFtw=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "176", + "X-Amzn-Trace-Id" : "Root=1-69e56657-703f689a2a7f6b3a604c4305;Parent=37dcde1dd4c47591;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "239", + "Date" : "Sun, 19 Apr 2026 23:33:43 GMT", + "Via" : "1.1 940972e9e344075576fe20d5db482122.cloudfront.net (CloudFront), 1.1 f8731007efc5ab360d90cee573a1e916.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e56657000000005b64dd2a3d922fd6", + "x-amzn-RequestId" : "c7162863-9923-4fb4-af36-4a0cdfc581d5", + "X-Amz-Cf-Id" : "yvASStGjvoGrgaAlijWjvP2_oUVuo0q4wcmOfZFnqiMu6SRw-IW7Fg==", + "Content-Type" : "application/json" + } + }, + "uuid" : "1a27eb77-e3d5-4987-ae9f-4c31512eb77e", + "persistent" : true, + "scenarioName" : "scenario-1-btql", + "requiredScenarioState" : "scenario-1-btql-2", + "insertionIndex" : 167 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json new file mode 100644 index 00000000..a6930aec --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json @@ -0,0 +1,42 @@ +{ + "id" : "6cc00d1f-84f6-4674-81b5-47f65f1c93e9", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"},\"op\":\"ne\"},\"left\":{\"right\":{\"value\":\"e4243df27a38be41de316d64dea6519a\",\"op\":\"literal\"},\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"},\"op\":\"eq\"},\"op\":\"and\"},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-6cc00d1f-84f6-4674-81b5-47f65f1c93e9.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFzt6ERjoAMEIBg=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "89", + "X-Amzn-Trace-Id" : "Root=1-69e56658-1b1bcdf05e29414e6126b4e5;Parent=69bf25d21314d7a0;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "151", + "Date" : "Sun, 19 Apr 2026 23:33:44 GMT", + "Via" : "1.1 05717f654525d5f71688fb57ace6362a.cloudfront.net (CloudFront), 1.1 d525041695bdb6325f78ebba5c11b8a2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e56658000000003e11ea79e72bbd9f", + "x-amzn-RequestId" : "e0b8bc76-d170-45ce-a85a-6caf92f40be5", + "X-Amz-Cf-Id" : "qKqAM_2v92fx_BcKMyTcxcQQ64Q9rv5mY5xtRPQgPEiz9W4P4i_JPg==", + "Content-Type" : "application/json" + } + }, + "uuid" : "6cc00d1f-84f6-4674-81b5-47f65f1c93e9", + "persistent" : true, + "insertionIndex" : 165 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json new file mode 100644 index 00000000..1a6e9003 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json @@ -0,0 +1,45 @@ +{ + "id" : "d0e9a3e5-918d-4a9b-a923-a3482ab7fb08", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"},\"op\":\"ne\"},\"left\":{\"right\":{\"value\":\"8b6742b1702306836d0410b9a48a7559\",\"op\":\"literal\"},\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"},\"op\":\"eq\"},\"op\":\"and\"},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-d0e9a3e5-918d-4a9b-a923-a3482ab7fb08.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFzo_FKnIAMEuOQ=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "68", + "X-Amzn-Trace-Id" : "Root=1-69e56639-1f6d015b51e924ce36b4f3a2;Parent=74ef17883725bb8f;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "140", + "Date" : "Sun, 19 Apr 2026 23:33:13 GMT", + "Via" : "1.1 940972e9e344075576fe20d5db482122.cloudfront.net (CloudFront), 1.1 83d24992402f7b214901ab76fbdc11e2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e5663900000000546be2742be4be7a", + "x-amzn-RequestId" : "cdaa4c72-f5d3-4217-bb84-5d6a006bfe4a", + "X-Amz-Cf-Id" : "5c9Y7GBqjVMrnCb84_xTLIA9kmKzpaQAYa3ybdJQMXMT4VBy38rE1w==", + "Content-Type" : "application/json" + } + }, + "uuid" : "d0e9a3e5-918d-4a9b-a923-a3482ab7fb08", + "persistent" : true, + "scenarioName" : "scenario-1-btql", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-btql-2", + "insertionIndex" : 169 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json new file mode 100644 index 00000000..231b067a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json @@ -0,0 +1,42 @@ +{ + "id" : "fce651bd-9cb7-4b5d-a488-5b3da8a28a16", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"},\"op\":\"ne\"},\"left\":{\"right\":{\"value\":\"46647b58b1eb1908a50cfc1db2d84f37\",\"op\":\"literal\"},\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"},\"op\":\"eq\"},\"op\":\"and\"},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-fce651bd-9cb7-4b5d-a488-5b3da8a28a16.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFztzGJAIAMEPRQ=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "204", + "X-Amzn-Trace-Id" : "Root=1-69e56657-599b7c60206b07ef47ac16a2;Parent=6af77456974bfe3d;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "295", + "Date" : "Sun, 19 Apr 2026 23:33:44 GMT", + "Via" : "1.1 20b3731a0ef4aba3db1fcd63c3ef2b2a.cloudfront.net (CloudFront), 1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e56657000000001747de1e7ae62526", + "x-amzn-RequestId" : "33a135dc-3298-4c68-b019-dc1b98300a30", + "X-Amz-Cf-Id" : "dD40RzTQh8l0NiM-JThFgtD7oeE38Lw_s0vsAHLgrO72m1tS9slMFg==", + "Content-Type" : "application/json" + } + }, + "uuid" : "fce651bd-9cb7-4b5d-a488-5b3da8a28a16", + "persistent" : true, + "insertionIndex" : 166 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-5279636b-7eb9-4959-a3be-de675f1f14e1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-5279636b-7eb9-4959-a3be-de675f1f14e1.json new file mode 100644 index 00000000..c563f0da --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-5279636b-7eb9-4959-a3be-de675f1f14e1.json @@ -0,0 +1,39 @@ +{ + "id" : "5279636b-7eb9-4959-a3be-de675f1f14e1", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CvUHCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4zLjEtYmNiODkxZAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBK9BgoYChZicmFpbnRydXN0LWF3cy1iZWRyb2NrEqAGChCA2hVehepKIEuEFdGx6AB0EghUfq2C3tMHKioXYmVkcm9jay5jb252ZXJzZS1zdHJlYW0wATm4F93ZGOanGEF6TM8aGeanGEqDAQoVYnJhaW50cnVzdC5pbnB1dF9qc29uEmoKaFt7InJvbGUiOiJ1c2VyIiwiY29udGVudCI6W3sidGV4dCI6IldoYXQgaXMgdGhlIGNhcGl0YWwgb2YgRnJhbmNlPyBSZXBseSBpbiBvbmUgd29yZC4iLCJ0eXBlIjoidGV4dCJ9XX1dSm0KEmJyYWludHJ1c3QubWV0cmljcxJXClV7ImNvbXBsZXRpb25fdG9rZW5zIjo1LCJwcm9tcHRfdG9rZW5zIjoxOSwidG9rZW5zIjoyNCwidGltZV90b19maXJzdF90b2tlbiI6MC4wMDIwMDF9SjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEouChpicmFpbnRydXN0LnNwYW5fYXR0cmlidXRlcxIQCg57InR5cGUiOiJsbG0ifUpeChZicmFpbnRydXN0Lm91dHB1dF9qc29uEkQKQlt7InJvbGUiOiJhc3Npc3RhbnQiLCJjb250ZW50IjpbeyJ0ZXh0IjoiUGFyaXMuIiwidHlwZSI6InRleHQifV19XUqTAgoTYnJhaW50cnVzdC5tZXRhZGF0YRL7AQr4AXsiZW5kcG9pbnQiOiJjb252ZXJzZS1zdHJlYW0iLCJwcm92aWRlciI6ImJlZHJvY2siLCJyZXF1ZXN0X3BhdGgiOiJtb2RlbC91cy5hbnRocm9waWMuY2xhdWRlLTMtaGFpa3UtMjAyNDAzMDctdjElM0EwL2NvbnZlcnNlLXN0cmVhbSIsIm1vZGVsIjoidXMuYW50aHJvcGljLmNsYXVkZS0zLWhhaWt1LTIwMjQwMzA3LXYxOjAiLCJyZXF1ZXN0X2Jhc2VfdXJpIjoiaHR0cDovL2xvY2FsaG9zdCIsInJlcXVlc3RfbWV0aG9kIjoiUE9TVCJ9egCFAQEBAAA=" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFzl-ETKIAMEjRg=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69e56625-441e8c8f2bd4320b17181c96;Parent=6a350a2a27093fd8;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Sun, 19 Apr 2026 23:32:54 GMT", + "Via" : "1.1 5e599a9eda8861379cfef6a522da18e4.cloudfront.net (CloudFront), 1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e566250000000005daf7e9b40cc191", + "x-amzn-RequestId" : "cce8cec1-50e4-4dc4-8324-63784d2a19a7", + "X-Amz-Cf-Id" : "KWmCJgrhWRcuHZOov8b-fY3YLFmZzoPvlOwnkXa1fWoAG9L3DKDicQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "5279636b-7eb9-4959-a3be-de675f1f14e1", + "persistent" : true, + "insertionIndex" : 164 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-57070a21-af08-425a-886a-32727f7339dd.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-57070a21-af08-425a-886a-32727f7339dd.json new file mode 100644 index 00000000..41489171 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-57070a21-af08-425a-886a-32727f7339dd.json @@ -0,0 +1,39 @@ +{ + "id" : "57070a21-af08-425a-886a-32727f7339dd", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "Cp4HCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4zLjEtYmNiODkxZAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBLmBQoYChZicmFpbnRydXN0LWF3cy1iZWRyb2NrEskFChAYsM6KEH7rtL/f/S38lbeJEghu2s+hzIfC/CoQYmVkcm9jay5jb252ZXJzZTABOegxkiQa5qcYQcnSQ1Qa5qcYSoMBChVicmFpbnRydXN0LmlucHV0X2pzb24SagpoW3sicm9sZSI6InVzZXIiLCJjb250ZW50IjpbeyJ0ZXh0IjoiV2hhdCBpcyB0aGUgY2FwaXRhbCBvZiBGcmFuY2U/IFJlcGx5IGluIG9uZSB3b3JkLiIsInR5cGUiOiJ0ZXh0In1dfV1KTgoSYnJhaW50cnVzdC5tZXRyaWNzEjgKNnsiY29tcGxldGlvbl90b2tlbnMiOjIsInByb21wdF90b2tlbnMiOjEyLCJ0b2tlbnMiOjE0fUoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KXQoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhJDCkFbeyJjb250ZW50IjpbeyJ0ZXh0IjoiUGFyaXMiLCJ0eXBlIjoidGV4dCJ9XSwicm9sZSI6ImFzc2lzdGFudCJ9XUrjAQoTYnJhaW50cnVzdC5tZXRhZGF0YRLLAQrIAXsiZW5kcG9pbnQiOiJjb252ZXJzZSIsInByb3ZpZGVyIjoiYmVkcm9jayIsInJlcXVlc3RfcGF0aCI6Im1vZGVsL3VzLmFtYXpvbi5ub3ZhLWxpdGUtdjElM0EwL2NvbnZlcnNlIiwibW9kZWwiOiJ1cy5hbWF6b24ubm92YS1saXRlLXYxOjAiLCJyZXF1ZXN0X2Jhc2VfdXJpIjoiaHR0cDovL2xvY2FsaG9zdCIsInJlcXVlc3RfbWV0aG9kIjoiUE9TVCJ9egCFAQEBAAA=" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFzmxHysoAMEmVA=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69e5662a-19ef64fa06e84fe52eca8f87;Parent=5611cc5c45c49577;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Sun, 19 Apr 2026 23:32:59 GMT", + "Via" : "1.1 e1832834d17ab65dd955f4e68cc524e6.cloudfront.net (CloudFront), 1.1 fbb003dfc0617e3e058e3dac791dfd5a.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e5662a0000000063fe6666688a9ed4", + "x-amzn-RequestId" : "d67ec18b-e026-4e21-ad72-7beda6bc696c", + "X-Amz-Cf-Id" : "QxuOoaWj89tHZf0bf0-Wv39xn9qEVx0EQ6348UBSiiOnEcS8De0NnQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "57070a21-af08-425a-886a-32727f7339dd", + "persistent" : true, + "insertionIndex" : 162 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-75dd8273-1f17-47fd-a992-66877387d37e.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-75dd8273-1f17-47fd-a992-66877387d37e.json new file mode 100644 index 00000000..a1f604b9 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-75dd8273-1f17-47fd-a992-66877387d37e.json @@ -0,0 +1,39 @@ +{ + "id" : "75dd8273-1f17-47fd-a992-66877387d37e", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CsEHCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4zLjEtYmNiODkxZAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBKJBgoYChZicmFpbnRydXN0LWF3cy1iZWRyb2NrEuwFChCy97QyFjPe8Eg2mLgJUP4IEgih7+cWMq5DnCoQYmVkcm9jay5jb252ZXJzZTABOdg79eUZ5qcYQTcCLgsa5qcYSoMBChVicmFpbnRydXN0LmlucHV0X2pzb24SagpoW3sicm9sZSI6InVzZXIiLCJjb250ZW50IjpbeyJ0ZXh0IjoiV2hhdCBpcyB0aGUgY2FwaXRhbCBvZiBGcmFuY2U/IFJlcGx5IGluIG9uZSB3b3JkLiIsInR5cGUiOiJ0ZXh0In1dfV1KTgoSYnJhaW50cnVzdC5tZXRyaWNzEjgKNnsiY29tcGxldGlvbl90b2tlbnMiOjUsInByb21wdF90b2tlbnMiOjE5LCJ0b2tlbnMiOjI0fUoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KXgoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhJECkJbeyJjb250ZW50IjpbeyJ0ZXh0IjoiUGFyaXMuIiwidHlwZSI6InRleHQifV0sInJvbGUiOiJhc3Npc3RhbnQifV1KhQIKE2JyYWludHJ1c3QubWV0YWRhdGES7QEK6gF7ImVuZHBvaW50IjoiY29udmVyc2UiLCJwcm92aWRlciI6ImJlZHJvY2siLCJyZXF1ZXN0X3BhdGgiOiJtb2RlbC91cy5hbnRocm9waWMuY2xhdWRlLTMtaGFpa3UtMjAyNDAzMDctdjElM0EwL2NvbnZlcnNlIiwibW9kZWwiOiJ1cy5hbnRocm9waWMuY2xhdWRlLTMtaGFpa3UtMjAyNDAzMDctdjE6MCIsInJlcXVlc3RfYmFzZV91cmkiOiJodHRwOi8vbG9jYWxob3N0IiwicmVxdWVzdF9tZXRob2QiOiJQT1NUIn16AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFzmlHGYIAMET-Q=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69e56629-5cd1a8c0794a4f6b3eb47a9e;Parent=2ee2105957db22b5;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Sun, 19 Apr 2026 23:32:57 GMT", + "Via" : "1.1 724581b48d733e53834b535d2a623034.cloudfront.net (CloudFront), 1.1 0df7f27a01014ab815259ca2d88193c6.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e566290000000055572a49559b620a", + "x-amzn-RequestId" : "adc86931-7fe8-499c-a5c4-f67bf1097ed5", + "X-Amz-Cf-Id" : "stArdbVvbIxGJ8ZYc66r7k32hj9GsNWmOMhDR34ZohRixKkaVYDLbQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "75dd8273-1f17-47fd-a992-66877387d37e", + "persistent" : true, + "insertionIndex" : 163 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-eb149792-2d60-4963-9808-9a8bf6326d31.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-eb149792-2d60-4963-9808-9a8bf6326d31.json new file mode 100644 index 00000000..b467743b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-eb149792-2d60-4963-9808-9a8bf6326d31.json @@ -0,0 +1,39 @@ +{ + "id" : "eb149792-2d60-4963-9808-9a8bf6326d31", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "Ct8fCrgBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAooCg9zZXJ2aWNlLnZlcnNpb24SFQoTMC4zLjEtYmNiODkxZC1ESVJUWQogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBLjGgoYChZicmFpbnRydXN0LWF3cy1iZWRyb2NrEqsIChCLZ0KxcCMGg20EELmkinVZEgjBIVqTS+fVhSIIBvjVeSIBUjoqEGJlZHJvY2suY29udmVyc2UwATmpvCKRHOanGEE9UevUHOanGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KjQIKFWJyYWludHJ1c3QuaW5wdXRfanNvbhLzAQrwAVt7InJvbGUiOiJ1c2VyIiwiY29udGVudCI6W3sidGV4dCI6IldoYXQgY29sb3IgaXMgdGhpcyBpbWFnZT8iLCJ0eXBlIjoidGV4dCJ9LHsiaW1hZ2UiOnsiZm9ybWF0IjoicG5nIiwic291cmNlIjp7ImJ5dGVzIjoiaVZCT1J3MEtHZ29BQUFBTlNVaEVVZ0FBQUFFQUFBQUJDQVlBQUFBZkZjU0pBQUFBRFVsRVFWUjQybVA4ejhEd0h3QUZCUUlBWDhqeDBnQUFBQUJKUlU1RXJrSmdnZz09In19LCJ0eXBlIjoiaW1hZ2UifV19XUpRChJicmFpbnRydXN0Lm1ldHJpY3MSOwo5eyJjb21wbGV0aW9uX3Rva2VucyI6NDksInByb21wdF90b2tlbnMiOjUzNiwidG9rZW5zIjo1ODV9SuMBChNicmFpbnRydXN0Lm1ldGFkYXRhEssBCsgBeyJlbmRwb2ludCI6ImNvbnZlcnNlIiwicHJvdmlkZXIiOiJiZWRyb2NrIiwicmVxdWVzdF9wYXRoIjoibW9kZWwvdXMuYW1hem9uLm5vdmEtbGl0ZS12MSUzQTAvY29udmVyc2UiLCJtb2RlbCI6InVzLmFtYXpvbi5ub3ZhLWxpdGUtdjE6MCIsInJlcXVlc3RfYmFzZV91cmkiOiJodHRwOi8vbG9jYWxob3N0IiwicmVxdWVzdF9tZXRob2QiOiJQT1NUIn1KpwIKFmJyYWludHJ1c3Qub3V0cHV0X2pzb24SjAIKiQJbeyJjb250ZW50IjpbeyJ0ZXh0IjoiU29ycnksIEkgY2FuJ3QgZGVzY3JpYmUgYW4gaW1hZ2UuIEkgY2FuIG9ubHkgcHJvdmlkZSBpbmZvcm1hdGlvbiBhYm91dCB0aGUgY29sb3IuIEhvd2V2ZXIsIGlmIHlvdSB3YW50IHRvIGtub3cgYWJvdXQgdGhlIGNvbG9yIG9mIHRoZSBpbWFnZSwgSSBjYW4gcHJvdmlkZSBpbmZvcm1hdGlvbiBhYm91dCBpdC4gVGhlIGltYWdlIGlzIGluIGEgcmVkIGNvbG9yLiIsInR5cGUiOiJ0ZXh0In1dLCJyb2xlIjoiYXNzaXN0YW50In1degCFAQEBAAAS7QoKEEZke1ix6xkIpQz8HbLYTzcSCGYIyAohvsrkIghffT6uZSEnXyoQYmVkcm9jay5jb252ZXJzZTABOZ3FIpEc5qcYQXgqit8c5qcYSjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEouChpicmFpbnRydXN0LnNwYW5fYXR0cmlidXRlcxIQCg57InR5cGUiOiJsbG0ifUpwChVicmFpbnRydXN0LmlucHV0X2pzb24SVwpVW3sicm9sZSI6InVzZXIiLCJjb250ZW50IjpbeyJ0ZXh0IjoiV2hhdCBpcyB0aGUgY2FwaXRhbCBvZiBGcmFuY2U/IiwidHlwZSI6InRleHQifV19XUpQChJicmFpbnRydXN0Lm1ldHJpY3MSOgo4eyJjb21wbGV0aW9uX3Rva2VucyI6MTQ0LCJwcm9tcHRfdG9rZW5zIjo3LCJ0b2tlbnMiOjE1MX1K4wEKE2JyYWludHJ1c3QubWV0YWRhdGESywEKyAF7ImVuZHBvaW50IjoiY29udmVyc2UiLCJwcm92aWRlciI6ImJlZHJvY2siLCJyZXF1ZXN0X3BhdGgiOiJtb2RlbC91cy5hbWF6b24ubm92YS1saXRlLXYxJTNBMC9jb252ZXJzZSIsIm1vZGVsIjoidXMuYW1hem9uLm5vdmEtbGl0ZS12MTowIiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHA6Ly9sb2NhbGhvc3QiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqIBgoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhLtBQrqBVt7ImNvbnRlbnQiOlt7InRleHQiOiJUaGUgY2FwaXRhbCBvZiBGcmFuY2UgaXMgUGFyaXMuIFBhcmlzIGlzIG5vdCBvbmx5IHRoZSBjYXBpdGFsIGJ1dCBhbHNvIHRoZSBsYXJnZXN0IGNpdHkgaW4gRnJhbmNlLiBJdCBpcyBzaXR1YXRlZCBpbiB0aGUgbm9ydGhlcm4gY2VudHJhbCBwYXJ0IG9mIHRoZSBjb3VudHJ5LCBhbG9uZyB0aGUgU2VpbmUgUml2ZXIuIFBhcmlzIGlzIHJlbm93bmVkIGZvciBpdHMgcmljaCBoaXN0b3J5LCBjdWx0dXJlLCBhbmQgbGFuZG1hcmtzLiBTb21lIG9mIGl0cyBtb3N0IGZhbW91cyBhdHRyYWN0aW9ucyBpbmNsdWRlIHRoZSBFaWZmZWwgVG93ZXIsIHRoZSBMb3V2cmUgTXVzZXVtLCBOb3RyZS1EYW1lIENhdGhlZHJhbCwgYW5kIHRoZSBDaGFtcHMtw4lseXPDqWVzLiBJdCBpcyBhIG1ham9yIGdsb2JhbCBjaXR5IGFuZCBhIGh1YiBmb3IgYXJ0LCBmYXNoaW9uLCBnYXN0cm9ub215LCBhbmQgZGlwbG9tYWN5LiBQYXJpcyBpcyBkaXZpZGVkIGludG8gMjAgYXJyb25kaXNzZW1lbnRzIChtdW5pY2lwYWxpdGllcyksIGVhY2ggd2l0aCBpdHMgb3duIHVuaXF1ZSBjaGFyYWN0ZXIgYW5kIGF0dHJhY3Rpb25zLiBUaGUgY2l0eSBpcyBhbHNvIGtub3duIGZvciBpdHMgc2lnbmlmaWNhbnQgY29udHJpYnV0aW9ucyB0byB2YXJpb3VzIGZpZWxkcyBzdWNoIGFzIHBoaWxvc29waHksIHNjaWVuY2UsIGFuZCBsaXRlcmF0dXJlLiIsInR5cGUiOiJ0ZXh0In1dLCJyb2xlIjoiYXNzaXN0YW50In1degCFAQEBAAASqAcKEOQkPfJ6OL5B3jFtZN6mUZoSCHGOL1Ta0e5mIggjV8FrY4jo4yoXYmVkcm9jay5jb252ZXJzZS1zdHJlYW0wATkyuyKRHOanGEHwJ50WHeanGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KZAoVYnJhaW50cnVzdC5pbnB1dF9qc29uEksKSVt7InJvbGUiOiJ1c2VyIiwiY29udGVudCI6W3sidGV4dCI6ImNvdW50IHRvIDEwIHNsb3dseSIsInR5cGUiOiJ0ZXh0In1dfV1KcAoSYnJhaW50cnVzdC5tZXRyaWNzEloKWHsiY29tcGxldGlvbl90b2tlbnMiOjcxLCJwcm9tcHRfdG9rZW5zIjo2LCJ0b2tlbnMiOjc3LCJ0aW1lX3RvX2ZpcnN0X3Rva2VuIjowLjAwMjYwMTgzM31K8QEKE2JyYWludHJ1c3QubWV0YWRhdGES2QEK1gF7ImVuZHBvaW50IjoiY29udmVyc2Utc3RyZWFtIiwicHJvdmlkZXIiOiJiZWRyb2NrIiwicmVxdWVzdF9wYXRoIjoibW9kZWwvdXMuYW1hem9uLm5vdmEtbGl0ZS12MSUzQTAvY29udmVyc2Utc3RyZWFtIiwibW9kZWwiOiJ1cy5hbWF6b24ubm92YS1saXRlLXYxOjAiLCJyZXF1ZXN0X2Jhc2VfdXJpIjoiaHR0cDovL2xvY2FsaG9zdCIsInJlcXVlc3RfbWV0aG9kIjoiUE9TVCJ9SpoCChZicmFpbnRydXN0Lm91dHB1dF9qc29uEv8BCvwBW3sicm9sZSI6ImFzc2lzdGFudCIsImNvbnRlbnQiOlt7InRleHQiOiJTdXJlLCBJJ2xsIGNvdW50IHRvIDEwIHNsb3dseSBmb3IgeW91OlxuXG4xLiBPbmVcbjIuIFR3b1xuMy4gVGhyZWVcbjQuIEZvdXJcbjUuIEZpdmVcbjYuIFNpeFxuNy4gU2V2ZW5cbjguIEVpZ2h0XG45LiBOaW5lXG4xMC4gVGVuXG5cblRoZXJlIHlvdSBnbyEgSWYgeW91IG5lZWQgYW55dGhpbmcgZWxzZSwgZmVlbCBmcmVlIHRvIGFzay4iLCJ0eXBlIjoidGV4dCJ9XX1degCFAQEBAAASuwMKBQoDYnR4Eo4BChCLZ0KxcCMGg20EELmkinVZEggG+NV5IgFSOioLYXR0YWNobWVudHMwATlQNMGIHOanGEGHnwHVHOanGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKEwoGY2xpZW50EgkKB2JlZHJvY2t6AIUBAQEAABKLAQoQRmR7WLHrGQilDPwdsthPNxIIX30+rmUhJ18qCGNvbnZlcnNlMAE5qE/BiBzmpxhBscmM3xzmpxhKMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0ShMKBmNsaWVudBIJCgdiZWRyb2NregCFAQEBAAASkgEKEOQkPfJ6OL5B3jFtZN6mUZoSCCNXwWtjiOjjKg9jb252ZXJzZV9zdHJlYW0wATloMMGIHOanGEHo8MCeHeanGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKEwoGY2xpZW50EgkKB2JlZHJvY2t6AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cFzpBEeSoAMEc-g=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69e56639-2b6eaac951e9bc691575c14e;Parent=6220ccf7b0a1a2bf;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Sun, 19 Apr 2026 23:33:13 GMT", + "Via" : "1.1 2c24d855455b80190edd9e2dcdca3ee8.cloudfront.net (CloudFront), 1.1 a53bab1af200813b8f27e3c0a28b4964.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69e5663900000000768fc4ee912537cc", + "x-amzn-RequestId" : "96b7e297-bb0f-40b3-aa53-34eb1707485a", + "X-Amz-Cf-Id" : "jVfE0hTszQmLIU5DgAThDfyFNGoOGjOnEz621Hc4AESibNA1wdXR9Q==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "eb149792-2d60-4963-9808-9a8bf6326d31", + "persistent" : true, + "insertionIndex" : 168 +} \ No newline at end of file