feat(llmobs): add annotateAgentManifest manual API to LLMObs SDK

dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java

@@ -41,6 +41,8 @@ public class DDLLMObsSpan implements LLMObsSpan {
   private static final String SPAN_KIND = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND;
   private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA;
   private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS;
+  private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + LLMObsTags.AGENT_MANIFEST;
+  private static final String MANUAL_FRAMEWORK = "manual";
   private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD =
       LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method";
   private static final String INSTRUMENTATION_METHOD_ANNOTATED = "annotated";
@@ -291,6 +293,86 @@ public void annotatePrompt(LLMObs.Prompt prompt) {
     span.setTag(PROMPT_TRACKING_INSTRUMENTATION_METHOD, INSTRUMENTATION_METHOD_ANNOTATED);
   }
 
+  @Override
+  public void annotateAgentManifest(LLMObs.AgentManifest manifest) {
+    if (finished || manifest == null) {
+      return;
+    }
+    if (!Tags.LLMOBS_AGENT_SPAN_KIND.equals(spanKind)) {
+      LOGGER.warn(
+          "dropping agent manifest on non-agent span kind; annotateAgentManifest is only supported for agent spans");
+      return;
+    }
+    // Read existing manifest (may be null on first call)
+    Object existing = span.getTag(AGENT_MANIFEST);
+    @SuppressWarnings("unchecked")
+    Map<String, Object> base =
+        (existing instanceof Map)
+            ? new LinkedHashMap<>((Map<String, Object>) existing)
+            : new LinkedHashMap<>();
+    mergeManifest(base, manifest);
+    base.put("framework", MANUAL_FRAMEWORK);
+    span.setTag(AGENT_MANIFEST, base);
+  }
+
+  private void mergeManifest(Map<String, Object> base, LLMObs.AgentManifest manifest) {
+    // name: new non-empty wins, else keep existing, else span name
+    String manifestName = manifest.getName();
+    if (manifestName != null && !manifestName.isEmpty()) {
+      base.put("name", manifestName);
+    } else if (!base.containsKey("name")) {
+      CharSequence spanName = span.getSpanName();
+      if (spanName != null && spanName.length() > 0) {
+        base.put("name", spanName.toString());
+      }
+    }
+    // instructions
+    String instructions = manifest.getInstructions();
+    if (instructions != null && !instructions.isEmpty()) {
+      base.put("instructions", instructions);
+    }
+    // model
+    String model = manifest.getModel();
+    if (model != null && !model.isEmpty()) {
+      base.put("model", model);
+    }
+    // model_settings: shallow merge
+    Map<String, Object> modelSettings = manifest.getModelSettings();
+    if (modelSettings != null && !modelSettings.isEmpty()) {
+      @SuppressWarnings("unchecked")
+      Map<String, Object> existingSettings =
+          (base.get("model_settings") instanceof Map)
+              ? new LinkedHashMap<>((Map<String, Object>) base.get("model_settings"))
+              : new LinkedHashMap<>();
+      existingSettings.putAll(modelSettings);
+      base.put("model_settings", existingSettings);
+    }
+    // tools: replace if non-null non-empty
+    List<LLMObs.AgentTool> tools = manifest.getTools();
+    if (tools != null && !tools.isEmpty()) {
+      List<Map<String, Object>> toolList = new ArrayList<>();
+      for (LLMObs.AgentTool tool : tools) {
+        String toolName = tool == null ? null : tool.getName();
+        if (toolName == null || toolName.isEmpty()) {
+          LOGGER.warn("agent manifest tool missing required name; skipping");
+          continue;
+        }
+        Map<String, Object> toolMap = new LinkedHashMap<>();
+        toolMap.put("name", toolName);
+        if (tool.getDescription() != null) {
+          toolMap.put("description", tool.getDescription());
+        }
+        if (tool.getParameters() != null && !tool.getParameters().isEmpty()) {
+          toolMap.put("parameters", new LinkedHashMap<>(tool.getParameters()));
+        }
+        toolList.add(toolMap);
+      }
+      if (!toolList.isEmpty()) {
+        base.put("tools", toolList);
+      }
+    }
+  }
+
   private static Map<String, Object> copyStringKeyedMap(Map<?, ?> source) {
     Map<String, Object> copy = new LinkedHashMap<>();
     for (Map.Entry<?, ?> entry : source.entrySet()) {

dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy

@@ -63,6 +63,7 @@ class DDLLMObsSpanTest  extends DDSpecification{
   private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output"
   private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA
   private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS
+  private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + "agent_manifest"
   private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD =
   LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method"
 
@@ -791,6 +792,229 @@ class DDLLMObsSpanTest  extends DDSpecification{
     innerSpan.getTag(LLMOBS_TAG_PREFIX + "owner") == "ml-platform"
   }
 
+  def "agent manifest full annotation sets correct tag"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def settings = [temperature: 0.7, max_tokens: 1024]
+    def params = [city: [type: "string"]]
+    def tools = [LLMObs.AgentTool.from("get_weather", "Look up weather", params)]
+    def manifest = LLMObs.AgentManifest.builder()
+      .name("travel_desk")
+      .instructions("Book travel.")
+      .model("gpt-4o")
+      .modelSettings(settings)
+      .tools(tools)
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    stored["name"] == "travel_desk"
+    stored["instructions"] == "Book travel."
+    stored["model"] == "gpt-4o"
+    stored["framework"] == "manual"
+    def ms = (Map) stored["model_settings"]
+    ms["temperature"] == 0.7
+    ms["max_tokens"] == 1024
+    def toolList = (List) stored["tools"]
+    toolList.size() == 1
+    toolList[0]["name"] == "get_weather"
+    toolList[0]["description"] == "Look up weather"
+    toolList[0]["parameters"] == [city: [type: "string"]]
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest name defaults to span name when not provided"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def manifest = LLMObs.AgentManifest.builder().instructions("Do something.").build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    stored["name"] == "my-agent"
+    stored["instructions"] == "Do something."
+    stored["framework"] == "manual"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest drops tool with null name"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def validTool = LLMObs.AgentTool.from("valid-tool")
+    def badTool = LLMObs.AgentTool.from(null)
+    def manifest = LLMObs.AgentManifest.builder()
+      .tools([badTool, validTool])
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    def toolList = (List) stored["tools"]
+    toolList.size() == 1
+    toolList[0]["name"] == "valid-tool"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest with empty tools list omits tools key"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def manifest = LLMObs.AgentManifest.builder()
+      .name("agent")
+      .tools([])
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    !stored.containsKey("tools")
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest on non-agent span is silently dropped"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "llm-span")
+    def manifest = LLMObs.AgentManifest.builder().name("agent").build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    innerSpan.getTag(AGENT_MANIFEST) == null
+
+    cleanup:
+    test.finish()
+  }
+
+  def "second annotateAgentManifest call merges with the first"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def first = LLMObs.AgentManifest.builder()
+      .name("first")
+      .instructions("v1 instructions")
+      .model("gpt-3.5")
+      .build()
+    def second = LLMObs.AgentManifest.builder()
+      .name("second")
+      .model("gpt-4o")
+      .build()
+
+    when:
+    test.annotateAgentManifest(first)
+    test.annotateAgentManifest(second)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    stored["name"] == "second"          // second call wins on name
+    stored["model"] == "gpt-4o"         // second call wins on model
+    stored["instructions"] == "v1 instructions"  // first call's instructions preserved
+    stored["framework"] == "manual"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "second annotateAgentManifest call merges model_settings"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def first = LLMObs.AgentManifest.builder()
+      .name("agent")
+      .modelSettings([temperature: 0.5, max_tokens: 512])
+      .build()
+    def second = LLMObs.AgentManifest.builder()
+      .modelSettings([temperature: 0.9, top_p: 0.95])
+      .build()
+
+    when:
+    test.annotateAgentManifest(first)
+    test.annotateAgentManifest(second)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    def ms = (Map) stored["model_settings"]
+    ms["temperature"] == 0.9    // second wins
+    ms["max_tokens"] == 512     // first preserved
+    ms["top_p"] == 0.95         // second adds
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest model_settings forwarded as-is"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def manifest = LLMObs.AgentManifest.builder()
+      .name("agent")
+      .modelSettings([temperature: 0.5, custom_key: "custom_val"])
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    def ms = (Map) stored["model_settings"]
+    ms["temperature"] == 0.5
+    ms["custom_key"] == "custom_val"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "annotateAgentManifest null manifest is ignored"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+
+    when:
+    test.annotateAgentManifest(null)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    innerSpan.getTag(AGENT_MANIFEST) == null
+
+    cleanup:
+    test.finish()
+  }
+
+  def "annotateAgentManifest after finish is silently ignored"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def manifest = LLMObs.AgentManifest.builder().name("agent").model("gpt-4o").build()
+
+    when:
+    test.finish()
+    test.annotateAgentManifest(manifest)
+
+    then:
+    noExceptionThrown()
+    def innerSpan = (AgentSpan) test.span
+    innerSpan.getTag(AGENT_MANIFEST) == null
+  }
+
   private LLMObsSpan llmObsSpan(String kind, name) {
     llmObsSpan(kind, name, null)
   }