Add BEAM memory benchmarks and improve ingestion and retrieval

ts/packages/aiclient/package.json

@@ -41,6 +41,7 @@
     "@typeagent/telemetry": "workspace:*",
     "async": "^3.2.5",
     "debug": "^4.4.0",
+    "gpt-tokenizer": "^2.9.0",
     "typechat": "^0.1.1"
   },
   "devDependencies": {

ts/packages/aiclient/src/embeddingInput.ts

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { encode } from "gpt-tokenizer/encoding/cl100k_base";
+
+// Keep a safety margin below the hosted embedding models' 8,192-token limit.
+export const maxEmbeddingInputTokens = 8_000;
+
+export function countEmbeddingTokens(text: string): number {
+    return encode(text, {
+        allowedSpecial: new Set(),
+        disallowedSpecial: new Set(),
+    }).length;
+}
+
+export function embeddingInputLimitError(
+    input: string | readonly string[],
+): string | undefined {
+    const texts = typeof input === "string" ? [input] : input;
+    for (let index = 0; index < texts.length; index++) {
+        const tokens = countEmbeddingTokens(texts[index]);
+        if (tokens > maxEmbeddingInputTokens) {
+            return `Embedding input[${index}] has ${tokens} tokens; maximum is ${maxEmbeddingInputTokens} (cl100k_base)`;
+        }
+    }
+    return undefined;
+}

ts/packages/aiclient/src/index.ts

@@ -3,6 +3,7 @@
 
 export * from "./common.js";
 export * from "./models.js";
+export * from "./embeddingInput.js";
 export {
     instrumentChatModel,
     resetLlmClassificationDiagnostics,

ts/packages/aiclient/src/openai.ts

@@ -30,6 +30,7 @@ import {
 import { discoverEndpointPoolFromConfig } from "./endpointPoolFromConfig.js";
 import { getRuntimeConfig } from "./runtimeConfig.js";
 import { identifyEmbeddingModel } from "./identifiedEmbedding.js";
+import { embeddingInputLimitError } from "./embeddingInput.js";
 import {
     PromptSection,
     Result,
@@ -1043,6 +1044,10 @@ export function createEmbeddingModel(
     }
 
     async function callApi(input: string | string[]): Promise<Result<unknown>> {
+        const inputError = embeddingInputLimitError(input);
+        if (inputError !== undefined) {
+            return error(inputError);
+        }
         const params = {
             ...defaultParams,
             input,

ts/packages/aiclient/src/wireApiProvider/chatCompletionsWireApiProvider.ts

@@ -209,6 +209,9 @@ export class ChatCompletionsWireApiProvider implements ProviderAdapter {
         if (!completion.choices || completion.choices.length === 0) {
             return error("No choices returned");
         }
+        if (completion.choices[0].finish_reason === "content_filter") {
+            return error("Model response blocked by content filter");
+        }
         if (Array.isArray(request.jsonSchema)) {
             const tool_calls = completion.choices[0].message?.tool_calls;
             if (tool_calls === undefined) {

ts/packages/aiclient/test/embeddingInput.spec.ts

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { jest } from "@jest/globals";
+import {
+    countEmbeddingTokens,
+    embeddingInputLimitError,
+    maxEmbeddingInputTokens,
+} from "../src/embeddingInput.js";
+import { createEmbeddingModel, ModelType } from "../src/openai.js";
+import type { AzureApiSettings } from "../src/azureSettings.js";
+
+const atLimit = " word".repeat(8_000);
+const aboveLimit = atLimit + " word";
+const settings: AzureApiSettings = {
+    provider: "azure",
+    modelType: ModelType.Embedding,
+    apiKey: "test-key",
+    endpoint:
+        "https://example.openai.azure.com/openai/deployments/ada-002/embeddings",
+};
+
+describe("embedding input token ceiling", () => {
+    afterEach(() => jest.restoreAllMocks());
+
+    test("measures exact limits and treats special-token spellings as literal text", () => {
+        expect(maxEmbeddingInputTokens).toBe(8_000);
+        expect(countEmbeddingTokens(atLimit)).toBe(8_000);
+        expect(countEmbeddingTokens(aboveLimit)).toBe(8_001);
+        expect(embeddingInputLimitError(atLimit)).toBeUndefined();
+        expect(embeddingInputLimitError(["short", aboveLimit])).toBe(
+            "Embedding input[1] has 8001 tokens; maximum is 8000 (cl100k_base)",
+        );
+        expect(() => countEmbeddingTokens("<|endoftext|>")).not.toThrow();
+    });
+
+    test.each(["single", "batch"])(
+        "rejects an oversized %s request before any HTTP/authentication work",
+        async (mode) => {
+            const fetch = jest
+                .spyOn(globalThis, "fetch")
+                .mockRejectedValue(new Error("No HTTP request should be made"));
+            const model = createEmbeddingModel(settings, 2);
+            const result =
+                mode === "single"
+                    ? await model.generateEmbedding(aboveLimit)
+                    : await model.generateEmbeddingBatch!([
+                          "short",
+                          aboveLimit,
+                      ]);
+            expect(result).toMatchObject({
+                success: false,
+                message: expect.stringContaining(
+                    "8001 tokens; maximum is 8000",
+                ),
+            });
+            expect(fetch).not.toHaveBeenCalled();
+        },
+    );
+
+    test.each(["single", "batch"])(
+        "sends the unmodified input at the exact ceiling through the %s API",
+        async (mode) => {
+            const fetch = jest
+                .spyOn(globalThis, "fetch")
+                .mockResolvedValue(
+                    Response.json({ data: [{ embedding: [1, 0] }] }),
+                );
+            const model = createEmbeddingModel(settings, 2);
+            const result =
+                mode === "single"
+                    ? await model.generateEmbedding(atLimit)
+                    : await model.generateEmbeddingBatch!([atLimit]);
+            expect(result.success).toBe(true);
+            expect(fetch).toHaveBeenCalledTimes(1);
+            const body = fetch.mock.calls[0][1]?.body;
+            expect(typeof body).toBe("string");
+            expect(JSON.parse(String(body)).input).toEqual(
+                mode === "single" ? atLimit : [atLimit],
+            );
+        },
+    );
+});

ts/packages/aiclient/test/providerAdapter.spec.ts

@@ -150,6 +150,42 @@ describe("ChatCompletionsWireApiProvider (default path is byte-identical)", () =
         expect(r.success).toBe(false);
     });
 
+    test.each([
+        { mode: "plain", request: makeRequest() },
+        {
+            mode: "structured",
+            request: makeRequest({
+                jsonSchema: { name: "Result", schema: { type: "object" } },
+            }),
+        },
+        {
+            mode: "function",
+            request: makeRequest({
+                jsonSchema: [{ type: "function", function: { name: "do_it" } }],
+            }),
+        },
+    ])("rejects content-filtered $mode responses", ({ request }) => {
+        for (const content of ["", "partial response"]) {
+            expect(
+                adapter.parseResponse(
+                    {
+                        id: "filtered",
+                        choices: [
+                            {
+                                finish_reason: "content_filter",
+                                message: { role: "assistant", content },
+                            },
+                        ],
+                    },
+                    request,
+                ),
+            ).toEqual({
+                success: false,
+                message: "Model response blocked by content filter",
+            });
+        }
+    });
+
     test("parseResponse unwraps a single tool call for function calling", () => {
         const req = makeRequest({
             jsonSchema: [{ type: "function", function: { name: "do_it" } }],

ts/packages/benchmarks/README.AUTOGEN.md

@@ -3,7 +3,7 @@
 
 <!-- AUTOGEN:DOCS:START -->
 
-<!-- AUTOGEN:DOCS:HASH:sha256=15762572bc3edad0debf9164134450cf42e4cd21465a1cd521cd9de9f64d0cdf -->
+<!-- AUTOGEN:DOCS:HASH:sha256=4c97b372aeb20ee28fb8cdaf948d6af8ba2e8c2fe1656c66ca378e13eae78c32 -->
 <!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
 
 # @typeagent/benchmarks — AI-generated documentation
@@ -12,7 +12,7 @@
 
 ## Overview
 
-TypeAgent translation bench: catalog, action-parameters grader, simple-action dataset synthesizer
+TypeAgent translation and conversation-memory benchmarks
 
 ## Reference
 
@@ -31,10 +31,11 @@ Workspace:
 - [@typeagent/action-schema](../../packages/actionSchema/README.md)
 - [@typeagent/agent-sdk](../../packages/agentSdk/README.md)
 - [@typeagent/aiclient](../../packages/aiclient/README.md)
+- [@typeagent/memory-service](../../packages/memory/service/README.md)
 - [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md)
 - [default-agent-provider](../../packages/defaultAgentProvider/README.md)
 
-External: `commander`, `gpt-tokenizer`, `js-yaml`, `zod`
+External: `commander`, `gpt-tokenizer`, `hyparquet`, `js-yaml`, `zod`
 
 ### Used by
 
@@ -52,10 +53,10 @@ _None._
 - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts)
 - [./src/core/fileJson.ts](./src/core/fileJson.ts)
 - [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json)
-- _…and 72 more under `./src/`._
+- _…and 83 more under `./src/`._
 
 ---
 
-_Auto-generated against commit `40f897f5db304fac05955332810b52380443c9d3` on `2026-09-28T15:15:05.190Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
+_Auto-generated against commit `73fbbd085b9f687f05f7b8036f0e2c66ac7f0355` on `2026-10-24T18:29:20.864Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
 
 <!-- AUTOGEN:DOCS:END -->

ts/packages/benchmarks/README.md

@@ -2,6 +2,12 @@
 
 Action-translation eval for TypeAgent: pinned catalogs, model prices, and scoring harness.
 
+## Conversation-memory benchmarks
+
+The [BEAM runbook](src/memoryBench/README.md) covers the TypeScript
+memory harness, offline tests, pinned protocol preparation, live execution and
+resume. It calls the durable memory service directly and requires no Python.
+
 ## Catalog + action-parameters grader
 
 Pinned `catalog.generated.json` and `action-parameters-grader.generated.json`. Code/script parameters use verify mode `llmAsAJudge` (not exact); synthesizer exclusions are derived from those fields. Regenerate with `pnpm run gen-catalog` (`--force` full rebuild). Tests: `pnpm run test:local`.

ts/packages/benchmarks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@typeagent/benchmarks",
   "version": "0.0.1",
-  "description": "TypeAgent translation bench: catalog, action-parameters grader, simple-action dataset synthesizer",
+  "description": "TypeAgent translation and conversation-memory benchmarks",
   "homepage": "https://github.com/microsoft/TypeAgent#readme",
   "repository": {
     "type": "git",
@@ -28,6 +28,7 @@
     "gen-catalog": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genCatalog.js && node --max-old-space-size=4096 dist/translationBench/scripts/genPolicy.js",
     "gen-policy": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genPolicy.js && node ./scripts/copyAssets.mjs",
     "jest-esm": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js",
+    "memory-bench": "node dist/memoryBench/cli.js",
     "merge-checkpoints": "pnpm run build && node dist/translationBench/scripts/mergeCheckpoints.js",
     "pick-eligible-actions": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/pickEligibleActions.js --model ${TB_PICKER_MODEL:-azure/gpt-5.6-sol} && node ./scripts/copyAssets.mjs",
     "prettier": "prettier --check package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore",
@@ -40,10 +41,12 @@
     "@typeagent/action-schema": "workspace:*",
     "@typeagent/agent-sdk": "workspace:*",
     "@typeagent/aiclient": "workspace:*",
+    "@typeagent/memory-service": "workspace:*",
     "agent-dispatcher": "workspace:*",
     "commander": "^12.1.0",
     "default-agent-provider": "workspace:*",
     "gpt-tokenizer": "^2.9.0",
+    "hyparquet": "^1.30.0",
     "js-yaml": "^4.3.2",
     "zod": "^4.1.13"
   },