diff --git a/Docs/MemoryHouseEnvelope/implementation-plan.md b/Docs/MemoryHouseEnvelope/implementation-plan.md new file mode 100644 index 0000000..5ca547f --- /dev/null +++ b/Docs/MemoryHouseEnvelope/implementation-plan.md @@ -0,0 +1,450 @@ +# MemoryHouse Envelope Implementation Plan + +## Purpose + +Implement the wrapping design in +[`memoryhouse-amt-envelope.md`](memoryhouse-amt-envelope.md) without +changing the existing Agent Memory Toolkit (AMT) record hierarchy or Cosmos DB +serving behavior. + +The implementation will serialize a complete `MemoryRecordBase` document into +a governed MemoryHouse envelope and reconstruct the typed AMT record from that +payload. Integration with a concrete MemoryHouse service will be isolated +behind protocols so the core adapter can be developed and tested before a +service SDK is selected. + +## Desired Outcome + +Applications can: + +1. Wrap any supported AMT record in a versioned MemoryHouse envelope. +2. Validate and unwrap the envelope into the correct AMT record subtype. +3. Apply explicit personal, team, or organization scope through `owner`. +4. Apply ACL policy independently from scope at the integration boundary. +5. Write, look up, scan, annotate, and forget envelopes through an abstract + MemoryHouse client. +6. Replicate envelopes and deletion tombstones reliably between MemoryHouse + and AMT serving stores. +7. Adopt MemoryHouse as the canonical store without breaking existing AMT + retrieval APIs. + +## Guiding Constraints + +- `record.to_doc()` is the authoritative serialized AMT payload. +- Wrapping must not mutate the source record or payload. +- Unwrapping must use `MemoryRecordBase.from_doc()` and preserve subtype + validation. +- Owner and ACL values must be explicit inputs or supplied by a trusted policy. +- `owner.type` is the canonical scope level; no separate serialized `scope` + field is introduced. +- `owner.id` identifies the user, team, or organization that owns the memory. +- `payload.user_id` remains an AMT subject and partition field, not a + MemoryHouse scope or authorization field. +- Annotations must not duplicate AMT payload fields. +- AMT subtype discovery uses subtype-specific content types. +- Unsupported content types and versions must fail explicitly. +- The first release must not alter existing `CosmosMemoryClient` behavior. +- Sync and async integrations must provide equivalent semantics. +- No MemoryHouse dependency should be added until an official client contract + is selected. + +## Proposed Package Structure + +```text +azure/cosmos/agent_memory/memoryhouse/ +├── __init__.py +├── models.py +├── adapter.py +├── protocols.py +├── service.py +├── aio/ +│ ├── __init__.py +│ └── service.py +└── projection.py + +tests/unit/memoryhouse/ +├── test_models.py +├── test_adapter.py +├── test_service.py +└── test_projection.py + +tests/unit/aio/memoryhouse/ +└── test_service.py +``` + +The initial implementation should include only `models.py`, `adapter.py`, and +`protocols.py`. Service and projection modules should be added in later phases. + +## Core Types + +### MemoryEnvelope + +Define Pydantic wire models matching the reviewed MemoryHouse contract: + +```python +class MemoryOwner(BaseModel): + type: Literal["user", "team", "organization"] + id: str + + @property + def scope(self) -> Literal["personal", "team", "organization"]: + return { + "user": "personal", + "team": "team", + "organization": "organization", + }[self.type] + + +class MemoryACL(BaseModel): + read: list[str] + write: list[str] + annotate: list[str] + forget: list[str] + + +class MemoryProvenance(BaseModel): + application: str + agent_id: str | None = Field(alias="agentId") + created_by: str = Field(alias="createdBy") + created_at: str = Field(alias="createdAt") + source_id: str = Field(alias="sourceId") + prompt_id: str | None = Field(default=None, alias="promptId") + prompt_version: str | None = Field(default=None, alias="promptVersion") + + +class MemoryEnvelope(BaseModel): + id: str + content_type: str = Field(alias="contentType") + owner: MemoryOwner + provenance: MemoryProvenance + acl: MemoryACL + annotations: dict[str, Any] + payload: dict[str, Any] +``` + +Model requirements: + +- Emit camel-case wire names where required. +- Reject unknown owner types. +- Reject a separate serialized `scope` field. +- Expose scope only as a non-serialized convenience property derived from + `owner.type`. +- Reject empty IDs and ACL principals. +- Require every ACL operation, even when its principal list is empty. +- Preserve unknown annotation namespaces. +- Treat the payload as a JSON object without interpreting it in the envelope + model. +- Use `extra="forbid"` for the standardized envelope and governance objects. + +If MemoryHouse publishes an official model package before implementation, +prefer those models and keep AMT-specific types limited to adapter inputs and +content-type mapping. + +## Adapter Design + +### Constants + +```python +AMT_MEMORY_CONTENT_TYPES = { + "turn": "application/vnd.microsoft.amt.turn+json;version=1", + "fact": "application/vnd.microsoft.amt.fact+json;version=1", + "episodic": "application/vnd.microsoft.amt.episodic+json;version=1", + "procedural": "application/vnd.microsoft.amt.procedural+json;version=1", + "thread_summary": ( + "application/vnd.microsoft.amt.thread-summary+json;version=1" + ), + "user_summary": ( + "application/vnd.microsoft.amt.user-summary+json;version=1" + ), +} +AMT_APPLICATION_NAME = "agent-memory-toolkit" +``` + +Content-type parsing should normalize insignificant whitespace but require an +exact supported subtype and version. + +### Wrapping + +`AMTMemoryEnvelopeAdapter.wrap()` will: + +1. Validate `store_id` as a safe path segment. +2. Serialize the AMT record with `record.to_doc()`. +3. Build the stable ID `amt/{store_id}/{record.id}`. +4. Require `owner`, `acl`, and `created_by`, or resolve them through an + injected trusted policy. +5. Build provenance from the record and explicit caller context. +6. Select the content type from the validated record type. +7. Copy optional caller-provided enrichment annotations without adding AMT + payload fields. +8. Return a validated `MemoryEnvelope`. + +The adapter must deep-copy the payload and annotations so later mutations do +not modify the source AMT record or previously returned envelopes. + +### Unwrapping + +`AMTMemoryEnvelopeAdapter.unwrap()` will: + +1. Validate the envelope content type and version. +2. Require a mapping payload. +3. Verify `provenance.sourceId` matches `payload.id`. +4. Verify the namespaced envelope ID ends with the expected encoded AMT ID. +5. Verify the content-type subtype agrees with `payload.type`. +6. Reconstruct the record with `MemoryRecordBase.from_doc(payload)`. +7. Return the typed `MemoryRecordBase` subtype. + +### Identifier Encoding + +Use percent encoding for `store_id` and AMT IDs rather than accepting arbitrary +slashes: + +```text +amt/{encoded-store-id}/{encoded-amt-id} +``` + +Provide `build_envelope_id()` and `parse_envelope_id()` helpers with round-trip +tests. Reject empty values and malformed prefixes. + +## MemoryHouse Client Protocol + +Define transport-neutral sync and async protocols: + +```python +class MemoryHouseClientProtocol(Protocol): + def lookup(self, memory_id: str) -> MemoryEnvelope: ... + def scan(self, query: MemoryScanQuery) -> list[MemoryEnvelope]: ... + def write(self, envelope: MemoryEnvelope) -> MemoryEnvelope: ... + def annotate( + self, + memory_id: str, + annotations: dict[str, Any], + ) -> MemoryEnvelope: ... + def forget(self, memory_id: str) -> None: ... +``` + +The async protocol exposes equivalent coroutine methods. Authentication and +authorization remain the responsibility of the MemoryHouse implementation; +the AMT integration must not provide a bypass or local authorization fallback. + +## Integration Service + +Add an opt-in `MemoryHouseService` after the core adapter is stable. It should: + +- Wrap and write AMT records. +- Look up and unwrap AMT envelopes. +- Scan envelopes and return typed AMT records with their envelopes. +- Preserve enrichment annotations when replacing an AMT payload. +- Expose annotation mutation for independently produced enrichment. +- Translate MemoryHouse not-found, conflict, authorization, and transport + errors into explicit AMT integration exceptions. + +Do not add these methods directly to `CosmosMemoryClient` in the first release. +Keeping the integration composable avoids coupling the established AMT public +API to an unfinalized MemoryHouse service contract. + +## Replication Strategy + +### Phase A: Adapter-Only + +AMT and MemoryHouse applications call the adapter explicitly. No automatic +replication or existing-client behavior changes occur. + +### Phase B: AMT-Authoritative Mirror + +For an initial production pilot, keep AMT Cosmos containers authoritative and +use their change feeds to: + +1. Read inserted or updated AMT documents. +2. Parse them with `MemoryRecordBase.from_doc()`. +3. Resolve owner scope and ACL through deployment policy. +4. Wrap and write them to MemoryHouse. +5. Record retry state and dead-letter terminal failures. + +This avoids unreliable synchronous dual writes and requires no change to AMT's +current write path. + +Deletes require explicit tombstones because Cosmos change feed behavior alone +may not provide all routing and governance information needed by the target. +Add a durable deletion event before enabling mirrored forget behavior. + +### Phase C: MemoryHouse-Authoritative + +After MemoryHouse write, audit, and availability requirements are proven: + +1. Route new governed writes through `MemoryHouseService`. +2. Publish a durable projection event in the same logical write workflow. +3. Project AMT payloads into the existing three Cosmos serving containers. +4. Continue serving AMT retrieval APIs from Cosmos. +5. Reconcile envelope and serving-store state periodically. + +The authority mode must be explicit configuration: + +```text +disabled +amt_mirror +memoryhouse_canonical +``` + +Startup validation must reject conflicting writer configurations that could +create replication loops. + +## Projection Rules + +The projector will: + +- Accept only supported AMT content types. +- Unwrap and validate the typed record before writing it to Cosmos. +- Verify the content-type subtype matches the validated payload `type`. +- Route by the validated payload `type`. +- Use the payload's `user_id` and `thread_id` as Cosmos partition keys. +- Upsert through existing `MemoryStore.upsert_memory()` behavior. +- Preserve the complete payload, including AMT lifecycle fields. +- Use idempotent envelope version or ETag checkpoints. +- Ignore replayed events that are not newer than the recorded checkpoint. + +Forgetting publishes a tombstone containing: + +```json +{ + "envelopeId": "amt/default/fact_123", + "sourceId": "fact_123", + "memoryType": "fact", + "userId": "u1", + "threadId": "thread-42" +} +``` + +The projector validates this data against its last known envelope before +deleting the AMT serving document. + +## Error Model + +Add integration-specific exceptions: + +- `MemoryEnvelopeValidationError` +- `UnsupportedMemoryContentTypeError` +- `MemoryEnvelopeIntegrityError` +- `MemoryHouseAuthorizationError` +- `MemoryHouseConflictError` +- `MemoryHouseTransportError` +- `MemoryProjectionError` + +Do not convert authorization or transport failures into empty results. +Projection failures must retain the envelope ID, operation, and retryability +classification without logging payload content. + +## Testing Strategy + +### Unit Tests + +- Envelope and governance model validation. +- Camel-case serialization and parsing. +- Wrap/unwrap round trips for all six AMT record types. +- Preservation of every payload field, including embeddings and subtype data. +- Stable and reversible envelope IDs. +- Empty annotations on wrapping unless enrichment is explicitly supplied. +- No AMT payload fields copied into annotations. +- Content-type and version rejection. +- Content-type subtype and payload-type mismatch detection. +- Provenance and payload integrity mismatch detection. +- Source-record immutability after wrapping. +- Sync and async service protocol behavior using fakes. + +### Property and Compatibility Tests + +- For every representative AMT document: + + ```python + unwrapped.to_doc() == original.to_doc() + ``` + +- Existing AMT model, store, retrieval, and pipeline tests remain unchanged. +- Historical AMT documents accepted by `MemoryRecordBase.from_doc()` also + survive a wrap/unwrap round trip. + +### Integration Tests + +- Write and lookup against a MemoryHouse test implementation. +- AMT Cosmos change-feed mirror with retry and replay. +- MemoryHouse-to-Cosmos projection routing for every memory type. +- Annotation update without payload mutation. +- Forget propagation and idempotent tombstone replay. +- Conflict behavior under concurrent payload and annotation updates. + +## Observability + +Emit structured telemetry for: + +- `memoryhouse.wrap` +- `memoryhouse.unwrap` +- `memoryhouse.write` +- `memoryhouse.lookup` +- `memoryhouse.annotate` +- `memoryhouse.forget` +- `memoryhouse.project` +- `memoryhouse.projection_retry` +- `memoryhouse.projection_dead_letter` + +Include envelope ID, AMT memory type, operation duration, result, and retry +count. Do not log payload content, embeddings, ACL contents, or owner IDs unless +the deployment's telemetry policy explicitly permits them. + +## Documentation and Public API + +When the adapter is released: + +- Export adapter and envelope types from + `azure.cosmos.agent_memory.memoryhouse`, not initially from the package root. +- Add API documentation and wrapping examples. +- Document identity and ACL policy requirements. +- Document supported content-type versions. +- Document authority modes before enabling replication. +- Add a changelog entry describing the feature as opt-in. + +## Delivery Phases + +### Phase 1: Contracts and Lossless Adapter + +Deliver models, adapter, exceptions, identifier helpers, and exhaustive unit +tests. This phase has no external service or Cosmos behavior changes. + +**Exit criteria:** all supported AMT record types round-trip losslessly and +invalid governance or content types fail explicitly. + +### Phase 2: Transport-Neutral Service Layer + +Deliver sync and async protocols, service implementations, fake clients, and +operation-level tests. + +**Exit criteria:** applications can use all five MemoryHouse operations through +an injected client without coupling AMT to a specific SDK. + +### Phase 3: AMT-Authoritative Mirror Pilot + +Deliver change-feed wrapping, ACL policy resolution, retries, dead-letter +handling, and reconciliation reporting. + +**Exit criteria:** AMT writes are mirrored reliably and replay produces no +duplicates or divergent IDs. + +### Phase 4: MemoryHouse-Canonical Projection + +Deliver durable projection events, Cosmos projection, forget tombstones, +authority-mode validation, and operational runbooks. + +**Exit criteria:** MemoryHouse can be the canonical governed store while all +existing AMT retrieval APIs continue to operate from consistent Cosmos +projections. + +## Open Decisions + +The following decisions must be resolved before Phase 2 or Phase 3: + +1. Whether MemoryHouse provides an official Python SDK and canonical models. +2. Exact scan-query and optimistic-concurrency contracts. +3. Identity principal syntax and the service responsible for ACL resolution. +4. The source of explicit team and organization scope during AMT mirroring. +5. Audit-event ownership and retention requirements. +6. Event or outbox technology used for canonical-store projection. +7. Whether payload embeddings are accepted by MemoryHouse storage policy. +8. Required consistency and recovery objectives for forget propagation. diff --git a/Docs/MemoryHouseEnvelope/memoryhouse-amt-envelope.md b/Docs/MemoryHouseEnvelope/memoryhouse-amt-envelope.md new file mode 100644 index 0000000..ae1a88f --- /dev/null +++ b/Docs/MemoryHouseEnvelope/memoryhouse-amt-envelope.md @@ -0,0 +1,403 @@ +# Wrapping Agent Memory Toolkit Records in a MemoryHouse Envelope + +## Overview + +Agent Memory Toolkit (AMT) and MemoryHouse standardize different layers of a +memory system: + +- AMT defines typed memory documents and product-specific retrieval fields. +- MemoryHouse defines a governed envelope around an opaque producer-owned + payload. + +The recommended integration is therefore to store the complete serialized AMT +document as the MemoryHouse payload. MemoryHouse fields provide ownership, +access control, provenance, discovery, enrichment, and audit without changing +the AMT representation. + +```text +MemoryHouse memory +├── ID +├── ContentType +├── Owner +├── Provenance +├── ACL +├── Annotations +└── Payload + └── Complete AMT document +``` + +This approach preserves AMT's typed records and allows existing AMT consumers +to reconstruct `TurnRecord`, `FactRecord`, `EpisodicRecord`, +`ProceduralRecord`, and summary records without losing information. + +## Design Goals + +- Preserve the AMT document without rewriting or normalizing its content. +- Add MemoryHouse governance without adding governance fields to every AMT + record type. +- Keep AMT's Cosmos containers and indexes available as serving projections. +- Allow MemoryHouse to discover AMT memories without understanding every AMT + subtype. +- Support future AMT schema versions through a versioned content type. +- Ensure MemoryHouse annotations can evolve without mutating the original AMT + payload. + +## Non-Goals + +- Replacing AMT's typed Pydantic models with a generic memory model. +- Making MemoryHouse interpret episodic or procedural payload structures. +- Moving AMT vector, full-text, ranking, or prompt-building behavior into the + MemoryHouse contract. +- Copying AMT payload fields into annotations for discovery. + +## Canonical Wrapper + +An AMT record is serialized with `MemoryRecordBase.to_doc()` and stored +unchanged under `payload`. + +```json +{ + "id": "amt/default/fact_123", + "contentType": "application/vnd.microsoft.amt.fact+json;version=1", + "owner": { + "type": "user", + "id": "u1" + }, + "provenance": { + "application": "agent-memory-toolkit", + "agentId": "architecture-agent", + "createdBy": "architecture-agent", + "createdAt": "2026-08-20T20:00:00Z", + "sourceId": "fact_123" + }, + "acl": { + "read": ["user:u1"], + "write": ["user:u1"], + "annotate": ["service:memory-enrichers"], + "forget": ["user:u1"] + }, + "annotations": {}, + "payload": { + "id": "fact_123", + "user_id": "u1", + "thread_id": "thread-42", + "role": "system", + "type": "fact", + "content": "The user prefers concise reviews.", + "metadata": { + "category": "preference" + }, + "salience": 0.8, + "confidence": 0.95, + "content_hash": "0123456789abcdef0123456789abcdef", + "prompt_id": "extract_memories.prompty", + "prompt_version": "v2", + "created_at": "2026-08-20T20:00:00Z" + } +} +``` + +## Field Mapping + +| AMT field | MemoryHouse location | Notes | +| --- | --- | --- | +| Complete `record.to_doc()` result | `payload` | Authoritative AMT artifact | +| `id` | `id` and `provenance.sourceId` | Envelope ID is namespaced; payload ID is unchanged | +| `user_id` | `owner.id` for personal scope | Team and organization scope require explicit writer context | +| `type` | `contentType` subtype and `payload.type` | Content type supports subtype discovery; payload remains authoritative | +| `agent_id` | `provenance.agentId` | May be absent on some AMT records | +| `created_at` | `provenance.createdAt` | Payload value remains authoritative for AMT | +| `prompt_id`, `prompt_version` | `provenance` extensions | Records the generating prompt when available | +| `thread_id`, `role`, `tags` | `payload` only | Product indexes may extract them outside the canonical envelope | +| `salience`, `confidence` | `payload` only | Do not duplicate producer ranking signals in annotations | +| Supersession fields | `payload` only | AMT remains authoritative for its lifecycle state | +| `embedding` | Payload or external reference annotation | MemoryHouse does not prescribe vector storage | +| Permissions | `acl` | Must be explicitly supplied or derived by trusted policy | + +Only fields required by the MemoryHouse governance contract should be repeated +outside the payload. AMT-specific retrieval fields stay exclusively in the +payload or in product-owned serving indexes. + +## Identifier Strategy + +MemoryHouse requires a unified ID space, while AMT IDs are generated within an +AMT deployment. Use a stable namespace: + +```text +amt/{store-id}/{amt-id} +``` + +Examples: + +```text +amt/default/fact_123 +amt/support-prod/ep_456 +amt/copilot-westus/proc_789 +``` + +The same AMT record must always produce the same envelope ID. Changing a +record's content does not create a new envelope ID unless AMT itself creates a +new record. + +## Content Type and Versioning + +Use subtype-specific vendor MIME types that identify the producer, AMT record +type, and schema version: + +```text +application/vnd.microsoft.amt.turn+json;version=1 +application/vnd.microsoft.amt.fact+json;version=1 +application/vnd.microsoft.amt.episodic+json;version=1 +application/vnd.microsoft.amt.procedural+json;version=1 +application/vnd.microsoft.amt.thread-summary+json;version=1 +application/vnd.microsoft.amt.user-summary+json;version=1 +``` + +The version describes the serialized AMT payload contract. The subtype in the +content type must agree with `payload.type`; consumers reject a mismatch. + +Consumers must reject unsupported versions rather than attempting a +best-effort parse. A future breaking AMT representation should introduce a new +content-type version and a corresponding adapter. + +## Adapter Contract + +Wrapping and unwrapping should be implemented outside the AMT record models so +MemoryHouse concerns do not become part of every AMT subtype. + +```python +class AMTMemoryEnvelopeAdapter: + def wrap( + self, + record: MemoryRecordBase, + *, + store_id: str, + owner: MemoryOwner, + acl: MemoryACL, + created_by: str, + ) -> MemoryEnvelope: + ... + + def unwrap(self, envelope: MemoryEnvelope) -> MemoryRecordBase: + ... +``` + +`wrap()` should: + +1. Serialize the record using `record.to_doc()`. +2. Generate the stable namespaced envelope ID. +3. Require an explicit owner and ACL. +4. Select the subtype-specific content type from the validated AMT record type. +5. Copy only caller-provided enrichment annotations. +6. Store the complete serialized document under `payload`. + +`unwrap()` should: + +1. Verify the caller has already passed MemoryHouse authorization. +2. Validate the content type and supported version. +3. Require a JSON object payload. +4. Verify the content-type subtype agrees with `payload.type`. +5. Reconstruct the typed record with `MemoryRecordBase.from_doc(payload)`. +6. Surface validation failures rather than silently returning an untyped + object. + +## Ownership and ACL Policy + +AMT's `user_id` identifies the user associated with a memory, but it is not an +authorization policy. The wrapper must not assume that possession of a +`user_id` grants access. + +### Scope Representation + +The `owner` object is the canonical scope representation. Do not add a separate +serialized `scope` field because it would duplicate `owner.type` and could +become inconsistent with the owner identity. + +| Memory scope | `owner.type` | `owner.id` | +| --- | --- | --- | +| Personal | `user` | User identity | +| Team | `team` | Team identity | +| Organization | `organization` | Organization identity | + +Examples: + +```json +{ + "owner": { + "type": "user", + "id": "u1" + } +} +``` + +```json +{ + "owner": { + "type": "team", + "id": "payments" + } +} +``` + +```json +{ + "owner": { + "type": "organization", + "id": "contoso" + } +} +``` + +The ownership fields have distinct responsibilities: + +- `owner.type` defines the scope level. +- `owner.id` identifies that scope. +- `provenance.createdBy` identifies the user or service that created the + memory. +- `acl` defines which callers may read, write, annotate, or forget the memory. +- `payload.user_id` retains AMT subject and partition semantics; it does not + define MemoryHouse scope or authorization. + +For existing AMT memories, a trusted policy can create a personal scope: + +```text +AMT user_id=u1 + ↓ +owner={type: user, id: u1} +``` + +Team and organization scopes require explicit context from the writer because +they cannot be inferred from an AMT document. ACL principals should use a +consistent namespace such as `user:`, `team:`, `organization:`, and `service:`. + +An SDK may expose a derived convenience property without adding it to the wire +format: + +```python +@property +def scope(self) -> Literal["personal", "team", "organization"]: + return { + "user": "personal", + "team": "team", + "organization": "organization", + }[self.owner.type] +``` + +## Annotation Policy + +Annotations must not repeat AMT payload fields. A newly wrapped AMT memory +normally starts with an empty annotation map: + +```json +{ + "annotations": {} +} +``` + +MemoryHouse enrichment pipelines should write their own named annotations: + +```json +{ + "annotations": { + "classification": { + "domain": "payments", + "sensitivity": "internal" + }, + "search": { + "embeddingReference": "embeddings/mem_123" + }, + "summary": { + "text": "User review-format preference." + } + } +} +``` + +Valid annotations are independently produced assertions such as +classification, generated summaries, or references to external indexes. AMT +fields such as `type`, `thread_id`, `role`, `tags`, `salience`, `confidence`, +and supersession state stay in the payload. + +Enrichers must not rewrite the payload. If the AMT record changes, the producer +replaces the payload while preserving enrichment annotations owned by other +writers. + +## Operation Mapping + +| MemoryHouse operation | AMT wrapper behavior | +| --- | --- | +| Lookup | Authorize, return the envelope, and optionally unwrap the AMT record | +| Scan | Filter on owner, content type, provenance, ACL, and annotations | +| Write | Wrap `record.to_doc()` and create or replace the envelope | +| Annotate | Modify named annotations without modifying the payload | +| Forget | Delete the envelope and propagate deletion to serving projections when configured | + +MemoryHouse lookup and scan events should be recorded by the MemoryHouse audit +layer. AMT logging is not a substitute for authorization or access audit. + +## Storage and Serving Architecture + +MemoryHouse should hold the governed canonical artifact. AMT's Cosmos +containers can remain product-specific serving projections optimized for +conversation replay, vector search, summaries, and procedural context. + +```text +AMT producer + │ + ▼ +AMTMemoryEnvelopeAdapter.wrap() + │ + ▼ +MemoryHouse canonical envelope + │ + ▼ change feed or outbox projector +AMT Cosmos serving containers and indexes +``` + +An asynchronous projector is preferred over uncoordinated synchronous dual +writes. It provides replay, observable failures, and a clear source of truth. +The projector validates that the content-type subtype agrees with +`payload.type`, then routes using the payload's `type`, `user_id`, and +`thread_id`. + +During an incremental migration, the existing AMT store may temporarily remain +authoritative. In that mode, changes should be captured through an outbox or +Cosmos change feed and wrapped into MemoryHouse. The deployment must explicitly +declare which store is authoritative to avoid update loops. + +## Updates, Supersession, and Forgetting + +An in-place AMT update replaces the payload under the same MemoryHouse ID. +MemoryHouse enrichment annotations should be preserved unless their producer +explicitly replaces them. + +AMT supersession remains exclusively in the payload. It is distinct from +MemoryHouse `Forget`: + +- Supersession preserves history and marks one AMT memory as replaced. +- Forget removes the governed artifact according to authorization and + retention policy. + +If AMT Cosmos containers are retained as serving projections, forgetting must +publish a tombstone containing the AMT ID and routing fields so the projector +can delete the corresponding document. + +## Failure Handling + +- Reject writes without an owner or ACL. +- Reject unsupported content-type versions. +- Reject malformed payloads before projecting them into AMT. +- Do not report a successful write when either the canonical MemoryHouse write + or its durable outbox record fails. +- Send projection failures to a retryable dead-letter path with the envelope + ID and operation. +- Use version or ETag checks to prevent concurrent updates from overwriting a + newer payload or annotation. + +## Recommendation + +Adopt the wrapper as an integration boundary rather than modifying AMT's +existing record hierarchy. The complete AMT document remains a lossless, +producer-owned payload, while MemoryHouse supplies the governance contract. +This lets AMT continue to evolve its memory types and serving strategy without +requiring changes to the MemoryHouse core. diff --git a/Docs/MemoryHouseEnvelope/tasks.md b/Docs/MemoryHouseEnvelope/tasks.md new file mode 100644 index 0000000..d32c609 --- /dev/null +++ b/Docs/MemoryHouseEnvelope/tasks.md @@ -0,0 +1,562 @@ +# MemoryHouse Envelope Implementation Tasks + +## Task Conventions + +- Tasks are ordered by dependency. +- **P0** is required for the lossless adapter milestone. +- **P1** is required for service integration. +- **P2** is required for production replication. +- A task is complete only when its acceptance criteria and tests pass. + +## Phase 1: Contracts and Lossless Adapter + +### MH-001 — Confirm the MemoryHouse wire contract + +**Priority:** P0 + +**Work** + +- Confirm required fields, JSON casing, owner values, owner-as-scope semantics, + ACL operations, annotation merge semantics, and content-type parsing rules. +- Determine whether an official MemoryHouse Python model package exists. +- Record any deviations from + [`memoryhouse-amt-envelope.md`](memoryhouse-amt-envelope.md). + +**Acceptance criteria** + +- The approved wire contract is documented. +- The implementation team knows whether to use official or local Pydantic + models. +- No unresolved field-name or required-field ambiguity remains. + +### MH-002 — Add the MemoryHouse integration package + +**Priority:** P0 +**Depends on:** MH-001 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/__init__.py` +- `azure/cosmos/agent_memory/memoryhouse/models.py` +- `azure/cosmos/agent_memory/memoryhouse/exceptions.py` + +**Work** + +- Create the package. +- Add or import `MemoryEnvelope`, `MemoryOwner`, `MemoryACL`, and + `MemoryProvenance`. +- Add AMT-specific integration exceptions. +- Configure camel-case aliases and strict governance validation. +- Add a non-serialized `scope` convenience property derived from `owner.type`. + +**Acceptance criteria** + +- Valid canonical examples parse and serialize to the approved wire shape. +- Missing owner, provenance, ACL, annotations, or payload fields are rejected. +- A serialized `scope` field is rejected. +- `owner.type` maps `user`, `team`, and `organization` to personal, team, and + organization scope. +- Unknown standardized envelope fields are rejected. +- Unknown annotation namespaces are preserved. + +### MH-003 — Implement envelope ID helpers + +**Priority:** P0 +**Depends on:** MH-002 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/adapter.py` +- `tests/unit/memoryhouse/test_adapter.py` + +**Work** + +- Implement `build_envelope_id(store_id, memory_id)`. +- Implement `parse_envelope_id(envelope_id)`. +- Percent-encode path segments. +- Reject empty IDs, malformed escapes, extra segments, and invalid prefixes. + +**Acceptance criteria** + +- IDs follow `amt/{encoded-store-id}/{encoded-amt-id}`. +- Every valid ID round-trips without information loss. +- Equivalent inputs always produce the same ID. + +### MH-004 — Implement AMT subtype content types + +**Priority:** P0 +**Depends on:** MH-002 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/adapter.py` + +**Work** + +- Define a versioned content type for every AMT record type. +- Implement content-type generation and parsing. +- Normalize AMT names such as `thread_summary` to MIME-safe subtype names. +- Reject unknown subtypes and versions. + +**Acceptance criteria** + +- Every supported AMT record type maps to exactly one content type. +- Content-type parsing returns the corresponding AMT record type. +- The mapping round-trips for all six record types. +- No AMT payload field is copied into annotations. + +### MH-005 — Implement `AMTMemoryEnvelopeAdapter.wrap` + +**Priority:** P0 +**Depends on:** MH-003, MH-004 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/adapter.py` +- `tests/unit/memoryhouse/test_adapter.py` + +**Work** + +- Define versioned content-type constants. +- Serialize with `record.to_doc()`. +- Build the envelope ID, owner, ACL, provenance, annotations, and payload. +- Require explicit `created_by`. +- Require explicit owner context unless a trusted personal-scope policy is + injected. +- Deep-copy mutable values. + +**Acceptance criteria** + +- Every supported AMT record subtype can be wrapped. +- The payload equals the source record's `to_doc()` result. +- Wrapping does not mutate the AMT record or caller annotations. +- Wrapping does not serialize a separate `scope` field. +- Prompt and source metadata are copied into provenance when present. + +### MH-006 — Implement `AMTMemoryEnvelopeAdapter.unwrap` + +**Priority:** P0 +**Depends on:** MH-005 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/adapter.py` +- `tests/unit/memoryhouse/test_adapter.py` + +**Work** + +- Validate media type and version. +- Verify envelope ID, provenance source ID, and payload ID. +- Verify the content-type subtype matches `payload.type`. +- Reconstruct records through `MemoryRecordBase.from_doc()`. +- Support explicit strict integrity checking. + +**Acceptance criteria** + +- Unwrapping returns the correct AMT subtype. +- Unsupported types and versions raise explicit errors. +- Malformed payloads and integrity mismatches are rejected. +- The payload remains authoritative when diagnostic mismatch reporting is used. + +### MH-007 — Add round-trip tests for all record types + +**Priority:** P0 +**Depends on:** MH-006 + +**Files** + +- `tests/unit/memoryhouse/test_adapter.py` +- `tests/unit/memoryhouse/test_models.py` + +**Work** + +- Cover turn, thread summary, user summary, fact, episodic, and procedural + records. +- Include optional fields, embeddings, lineage, supersession, tags, and subtype + structures. +- Test historical document shapes already supported by AMT. + +**Acceptance criteria** + +For every fixture: + +```python +adapter.unwrap(adapter.wrap(record, ...)).to_doc() == record.to_doc() +``` + +- No existing AMT tests need behavior changes. +- Negative tests cover every adapter exception. + +### MH-008 — Publish the adapter API and documentation + +**Priority:** P0 +**Depends on:** MH-007 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/__init__.py` +- `Docs/public_api.md` +- `README.md` +- `CHANGELOG.md` + +**Work** + +- Export integration types from the `memoryhouse` subpackage. +- Add a minimal wrapping and unwrapping example. +- Document that authorization occurs before unwrapping. +- Document supported content-type versions. + +**Acceptance criteria** + +- Users can import the adapter from + `azure.cosmos.agent_memory.memoryhouse`. +- Documentation does not imply that `user_id` is an ACL. +- The feature is documented as opt-in. + +## Phase 2: MemoryHouse Service Integration + +### MH-009 — Define sync and async client protocols + +**Priority:** P1 +**Depends on:** MH-001, MH-008 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/protocols.py` + +**Work** + +- Define lookup, scan, write, annotate, and forget contracts. +- Define scan filters and pagination representation. +- Define optimistic-concurrency inputs and outputs. +- Define equivalent async protocols. + +**Acceptance criteria** + +- Protocols can be satisfied by fakes without importing a concrete SDK. +- Sync and async operations have equivalent result and error semantics. +- Pagination and conflict behavior are unambiguous. + +### MH-010 — Add fake MemoryHouse clients + +**Priority:** P1 +**Depends on:** MH-009 + +**Files** + +- `tests/unit/memoryhouse/fakes.py` + +**Work** + +- Implement deterministic in-memory sync and async fakes. +- Enforce ID uniqueness, version conflicts, annotation merges, and forget. +- Record calls for assertions. + +**Acceptance criteria** + +- Fakes satisfy the protocols. +- Tests can simulate authorization, not-found, conflict, and transport errors. + +### MH-011 — Implement the sync MemoryHouse service + +**Priority:** P1 +**Depends on:** MH-010 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/service.py` +- `tests/unit/memoryhouse/test_service.py` + +**Work** + +- Implement wrapped write, typed lookup, scan, annotate, and forget. +- Preserve enrichment annotations during producer payload updates. +- Keep annotations limited to independently produced enrichment. +- Map concrete-client failures to integration exceptions. + +**Acceptance criteria** + +- All five operations work through the fake client. +- Authorization and transport errors are never returned as empty results. +- Payload replacement does not erase enrichment annotations. + +### MH-012 — Implement the async MemoryHouse service + +**Priority:** P1 +**Depends on:** MH-011 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/aio/__init__.py` +- `azure/cosmos/agent_memory/memoryhouse/aio/service.py` +- `tests/unit/aio/memoryhouse/test_service.py` + +**Work** + +- Implement native async operations. +- Match sync validation, error mapping, and annotation behavior. +- Do not use thread-pool wrappers for service calls. + +**Acceptance criteria** + +- Async tests cover parity with every sync operation and failure mode. +- No blocking transport calls run on the event loop. + +### MH-013 — Integrate the selected MemoryHouse SDK + +**Priority:** P1 +**Depends on:** MH-009, official SDK decision + +**Work** + +- Add an optional dependency only if required. +- Implement sync and async protocol adapters. +- Configure authentication externally. +- Add contract tests against a test service or emulator. + +**Acceptance criteria** + +- The SDK adapter passes the same contract suite as the fake clients. +- Credentials are not stored in envelopes, logs, or configuration files. +- Package users who do not use MemoryHouse do not need the optional dependency. + +## Phase 3: AMT-Authoritative Mirror + +### MH-014 — Define scope, owner, and ACL resolution policy + +**Priority:** P2 +**Depends on:** MH-001 + +**Work** + +- Define the trusted interface that maps an AMT document and deployment context + to `MemoryOwner` and `MemoryACL`. +- Define `owner.type` as the canonical personal, team, or organization scope. +- Define user, team, organization, and service principal syntax. +- Define failure behavior when policy cannot resolve governance. + +**Acceptance criteria** + +- Existing AMT records may map `user_id` to personal scope through trusted + policy. +- Team and organization scope never relies on inference from `user_id`. +- Creator identity is recorded in provenance and is not conflated with scope. +- Unresolved ownership or ACL policy blocks mirroring and produces an + actionable failure. +- Policy decisions are testable without a live identity provider. + +### MH-015 — Design the change-feed mirror checkpoint + +**Priority:** P2 +**Depends on:** MH-013, MH-014 + +**Work** + +- Define event identity, ordering, retry state, and idempotency keys. +- Account for AMT's three Cosmos containers. +- Define dead-letter records without copying memory payload content. +- Define operational replay controls. + +**Acceptance criteria** + +- Replaying an event cannot create a second envelope ID. +- Updates cannot overwrite a newer mirrored envelope. +- Operators can identify and retry failed envelope IDs. + +### MH-016 — Implement AMT-to-MemoryHouse mirroring + +**Priority:** P2 +**Depends on:** MH-015 + +**Files** + +- New Function App trigger or dedicated projector package +- Unit and integration tests + +**Work** + +- Consume changes from turns, memories, and summaries containers. +- Parse documents with `MemoryRecordBase.from_doc()`. +- Resolve governance, wrap, and write envelopes. +- Add bounded retries and dead-letter handling. + +**Acceptance criteria** + +- Every AMT memory type is mirrored. +- Replay is idempotent. +- Invalid AMT documents are dead-lettered with a safe diagnostic. +- Existing AMT write latency is unaffected. + +### MH-017 — Add mirror reconciliation + +**Priority:** P2 +**Depends on:** MH-016 + +**Work** + +- Scan AMT documents and expected envelope IDs. +- Detect missing, stale, and malformed mirrored envelopes. +- Support report-only and repair modes. + +**Acceptance criteria** + +- Report-only mode performs no writes. +- Repair mode is idempotent and uses optimistic concurrency. +- Metrics expose lag, failures, and divergence counts. + +### MH-018 — Implement delete and forget tombstones + +**Priority:** P2 +**Depends on:** MH-015 + +**Work** + +- Define a durable tombstone schema with AMT routing fields. +- Emit tombstones before deleting authoritative AMT documents where required. +- Propagate authorized forget operations. +- Make tombstone replay idempotent. + +**Acceptance criteria** + +- Forget reaches all configured stores. +- Replayed tombstones do not fail if the target is already absent. +- Deletion routing is validated against the last known envelope. + +## Phase 4: MemoryHouse-Canonical Projection + +### MH-019 — Add authority-mode configuration + +**Priority:** P2 +**Depends on:** MH-016 + +**Work** + +- Add `disabled`, `amt_mirror`, and `memoryhouse_canonical` modes. +- Validate configuration at startup. +- Prevent AMT mirror and canonical projection from forming an update loop. + +**Acceptance criteria** + +- Invalid or conflicting configurations fail startup. +- The default remains `disabled`. +- Existing deployments retain current behavior without configuration changes. + +### MH-020 — Define durable projection events + +**Priority:** P2 +**Depends on:** MH-019 + +**Work** + +- Define upsert and forget event schemas. +- Include envelope version or ETag and AMT routing data. +- Select the outbox or event transport. +- Define retry and dead-letter behavior. + +**Acceptance criteria** + +- A successful canonical write always has a durable projection event. +- Events are safe to replay and contain no credentials. +- Ordering and stale-event rejection rules are documented. + +### MH-021 — Implement MemoryHouse-to-AMT projection + +**Priority:** P2 +**Depends on:** MH-020 + +**Files** + +- `azure/cosmos/agent_memory/memoryhouse/projection.py` +- Projector host and tests + +**Work** + +- Validate and unwrap envelopes. +- Route records using the validated payload type. +- Upsert through existing `MemoryStore` behavior. +- Process forget tombstones. +- Checkpoint envelope versions. + +**Acceptance criteria** + +- All six AMT types reach the correct Cosmos container. +- Partition keys come from the validated payload. +- Replayed and stale events do not corrupt serving state. +- Existing retrieval APIs return projected records unchanged. + +### MH-022 — Add canonical-mode client integration + +**Priority:** P2 +**Depends on:** MH-021 + +**Work** + +- Decide whether to add a new governed client or opt-in methods on existing + clients. +- Route canonical writes through `MemoryHouseService`. +- Preserve existing direct-Cosmos methods for compatibility unless explicitly + deprecated. +- Surface projection status separately from canonical write success. + +**Acceptance criteria** + +- No existing API changes behavior unless canonical mode is explicitly enabled. +- Canonical write and projection status are distinguishable. +- Sync and async clients provide equivalent behavior. + +### MH-023 — Add end-to-end reliability tests + +**Priority:** P2 +**Depends on:** MH-022 + +**Work** + +- Test write, update, annotate, lookup, scan, supersede, and forget. +- Test process crashes between canonical write and projection. +- Test retries, duplicate events, stale events, and unavailable targets. +- Test recovery from dead-letter and reconciliation repair. + +**Acceptance criteria** + +- No acknowledged canonical write is permanently lost from projection. +- Replay does not create duplicates. +- Forget is eventually reflected in every configured serving store. +- Existing AMT retrieval and processing pipelines continue to pass. + +### MH-024 — Add operational documentation and rollout gates + +**Priority:** P2 +**Depends on:** MH-023 + +**Files** + +- `Docs/operations.md` +- `Docs/troubleshooting.md` +- `Docs/MemoryHouseEnvelope/` + +**Work** + +- Document configuration, identity, monitoring, replay, dead-letter recovery, + reconciliation, and rollback. +- Define pilot and production rollout metrics. +- Define procedures for switching authority modes. + +**Acceptance criteria** + +- Operators can diagnose mirror lag and failed projections. +- Rollback does not require deleting canonical memories. +- Production enablement requires passing documented reliability gates. + +## Cross-Cutting Validation + +Run the smallest relevant checks during each task and the complete suite at +phase boundaries: + +```bash +ruff check azure/cosmos/agent_memory/memoryhouse tests/unit/memoryhouse +pytest -q tests/unit/memoryhouse tests/unit/aio/memoryhouse +pytest -q tests/unit +``` + +Integration phases must also run their targeted live-service tests using the +project's existing `integration`, `slow`, and `e2e` markers. diff --git a/azure/cosmos/agent_memory/memoryhouse/__init__.py b/azure/cosmos/agent_memory/memoryhouse/__init__.py new file mode 100644 index 0000000..d2ad3b3 --- /dev/null +++ b/azure/cosmos/agent_memory/memoryhouse/__init__.py @@ -0,0 +1,41 @@ +"""MemoryHouse envelope integration for Agent Memory Toolkit records.""" + +from azure.cosmos.agent_memory.memoryhouse.adapter import ( + AMT_APPLICATION_NAME, + AMT_MEMORY_CONTENT_TYPES, + AMTMemoryEnvelopeAdapter, + build_envelope_id, + content_type_for_memory_type, + memory_type_for_content_type, + parse_envelope_id, +) +from azure.cosmos.agent_memory.memoryhouse.exceptions import ( + MemoryEnvelopeError, + MemoryEnvelopeIntegrityError, + MemoryEnvelopeValidationError, + UnsupportedMemoryContentTypeError, +) +from azure.cosmos.agent_memory.memoryhouse.models import ( + MemoryACL, + MemoryEnvelope, + MemoryOwner, + MemoryProvenance, +) + +__all__ = [ + "AMT_APPLICATION_NAME", + "AMT_MEMORY_CONTENT_TYPES", + "AMTMemoryEnvelopeAdapter", + "MemoryACL", + "MemoryEnvelope", + "MemoryEnvelopeError", + "MemoryEnvelopeIntegrityError", + "MemoryEnvelopeValidationError", + "MemoryOwner", + "MemoryProvenance", + "UnsupportedMemoryContentTypeError", + "build_envelope_id", + "content_type_for_memory_type", + "memory_type_for_content_type", + "parse_envelope_id", +] diff --git a/azure/cosmos/agent_memory/memoryhouse/adapter.py b/azure/cosmos/agent_memory/memoryhouse/adapter.py new file mode 100644 index 0000000..20388ce --- /dev/null +++ b/azure/cosmos/agent_memory/memoryhouse/adapter.py @@ -0,0 +1,242 @@ +"""Lossless wrapping of AMT records in MemoryHouse envelopes. + +The adapter deliberately duplicates only the minimum fields required by the +MemoryHouse contract: + +* The envelope ID repeats the AMT ID inside a globally namespaced identifier. +* The content type repeats the AMT record type for format discovery. +* Provenance repeats the source ID and selected writer metadata. + +All semantic memory data remains in the opaque payload. In particular, AMT +tags, confidence, salience, thread information, and lifecycle state are not +copied into annotations. +""" + +from __future__ import annotations + +import copy +import re +from collections.abc import Mapping +from typing import Any +from urllib.parse import quote, unquote + +from pydantic import ValidationError as PydanticValidationError + +from azure.cosmos.agent_memory.memoryhouse.exceptions import ( + MemoryEnvelopeIntegrityError, + MemoryEnvelopeValidationError, + UnsupportedMemoryContentTypeError, +) +from azure.cosmos.agent_memory.memoryhouse.models import ( + MemoryACL, + MemoryEnvelope, + MemoryOwner, + MemoryProvenance, +) +from azure.cosmos.agent_memory.models import MemoryRecordBase + +AMT_APPLICATION_NAME = "agent-memory-toolkit" + +# A subtype-specific media type lets MemoryHouse scan by broad artifact format +# without introducing a duplicate ``annotations.amt.memoryType`` field. +AMT_MEMORY_CONTENT_TYPES: dict[str, str] = { + "turn": "application/vnd.microsoft.amt.turn+json;version=1", + "fact": "application/vnd.microsoft.amt.fact+json;version=1", + "episodic": "application/vnd.microsoft.amt.episodic+json;version=1", + "procedural": "application/vnd.microsoft.amt.procedural+json;version=1", + "thread_summary": "application/vnd.microsoft.amt.thread-summary+json;version=1", + "user_summary": "application/vnd.microsoft.amt.user-summary+json;version=1", +} + +# Reverse lookup is precomputed so parsing and validation use the same +# authoritative mapping as serialization. +_CONTENT_TYPE_TO_MEMORY_TYPE = { + content_type: memory_type for memory_type, content_type in AMT_MEMORY_CONTENT_TYPES.items() +} + +# Insignificant whitespace around the semicolon and equals sign is accepted, +# but extra parameters and unsupported versions are rejected. +_CONTENT_TYPE_PATTERN = re.compile(r"^\s*([^;\s]+)\s*;\s*version\s*=\s*([^;\s]+)\s*$") + +# ``urllib.parse.unquote`` tolerates malformed percent escapes by leaving them +# unchanged. Detect them first so IDs cannot acquire multiple textual forms. +_INVALID_PERCENT_ESCAPE = re.compile(r"%(?![0-9A-Fa-f]{2})") + + +def _require_identifier(value: str, *, name: str) -> str: + """Require a usable identifier while preserving its exact spelling.""" + if not isinstance(value, str) or not value.strip(): + raise MemoryEnvelopeValidationError(f"{name} must be a non-empty string") + return value + + +def build_envelope_id(store_id: str, memory_id: str) -> str: + """Build the canonical MemoryHouse ID for an AMT record. + + Both components are encoded as path segments. This allows existing AMT IDs + to contain slashes, spaces, or percent characters without changing their + identity or adding ambiguous path levels. + """ + store_id = _require_identifier(store_id, name="store_id") + memory_id = _require_identifier(memory_id, name="memory_id") + return f"amt/{quote(store_id, safe='')}/{quote(memory_id, safe='')}" + + +def parse_envelope_id(envelope_id: str) -> tuple[str, str]: + """Parse a canonical AMT MemoryHouse ID into store and memory IDs.""" + envelope_id = _require_identifier(envelope_id, name="envelope_id") + parts = envelope_id.split("/") + if len(parts) != 3 or parts[0] != "amt": + raise MemoryEnvelopeValidationError("envelope_id must use the canonical 'amt/{store-id}/{memory-id}' format") + encoded_store_id, encoded_memory_id = parts[1:] + if not encoded_store_id or not encoded_memory_id: + raise MemoryEnvelopeValidationError("envelope_id store and memory segments must not be empty") + if _INVALID_PERCENT_ESCAPE.search(encoded_store_id) or _INVALID_PERCENT_ESCAPE.search(encoded_memory_id): + raise MemoryEnvelopeValidationError("envelope_id contains an invalid percent escape") + + store_id = unquote(encoded_store_id) + memory_id = unquote(encoded_memory_id) + + # Rebuilding the value enforces one canonical representation. For example, + # an unescaped space or a lower-case percent escape is rejected rather than + # being accepted as an alias for another envelope ID. + if build_envelope_id(store_id, memory_id) != envelope_id: + raise MemoryEnvelopeValidationError("envelope_id is not canonically encoded") + return store_id, memory_id + + +def content_type_for_memory_type(memory_type: str) -> str: + """Return the versioned MemoryHouse content type for an AMT record type.""" + try: + return AMT_MEMORY_CONTENT_TYPES[memory_type] + except KeyError as exc: + raise UnsupportedMemoryContentTypeError(f"Unsupported AMT memory type: {memory_type!r}") from exc + + +def memory_type_for_content_type(content_type: str) -> str: + """Parse a supported MemoryHouse content type into its AMT record type.""" + if not isinstance(content_type, str): + raise UnsupportedMemoryContentTypeError("Memory content type must be a string") + match = _CONTENT_TYPE_PATTERN.fullmatch(content_type) + if match is None: + raise UnsupportedMemoryContentTypeError(f"Unsupported memory content type: {content_type!r}") + normalized = f"{match.group(1)};version={match.group(2)}" + try: + return _CONTENT_TYPE_TO_MEMORY_TYPE[normalized] + except KeyError as exc: + raise UnsupportedMemoryContentTypeError(f"Unsupported memory content type: {content_type!r}") from exc + + +class AMTMemoryEnvelopeAdapter: + """Convert between typed AMT records and governed MemoryHouse envelopes. + + Authorization is intentionally outside this adapter. MemoryHouse must apply + the envelope ACL before a caller is allowed to obtain and unwrap a payload. + """ + + def wrap( + self, + record: MemoryRecordBase, + *, + store_id: str, + owner: MemoryOwner | Mapping[str, Any], + acl: MemoryACL | Mapping[str, Any], + created_by: str, + annotations: Mapping[str, Any] | None = None, + ) -> MemoryEnvelope: + if not isinstance(record, MemoryRecordBase): + raise MemoryEnvelopeValidationError("record must be a MemoryRecordBase instance") + + try: + # Validate governance separately from the AMT payload. ``user_id`` + # may supply personal ownership through a trusted caller policy, + # but this adapter never infers team or organization ownership. + validated_owner = MemoryOwner.model_validate(owner) + validated_acl = MemoryACL.model_validate(acl) + + # ``to_doc`` is the canonical AMT wire representation. A deep copy + # prevents mutations to the returned envelope from changing nested + # dictionaries or lists owned by the source record. + payload = copy.deepcopy(record.to_doc()) + provenance = MemoryProvenance( + application=AMT_APPLICATION_NAME, + agentId=record.agent_id, + createdBy=created_by, + createdAt=record.created_at, + sourceId=record.id, + promptId=record.prompt_id, + promptVersion=record.prompt_version, + ) + + # Caller annotations are accepted only as independent enrichment. + # The adapter adds no AMT discovery projection because the subtype + # content type and opaque payload already carry that information. + return MemoryEnvelope( + id=build_envelope_id(store_id, record.id), + # Read the wire discriminator from the serialized payload + # instead of stringifying the model attribute. Literal enum + # fields may stringify as ``MemoryType.fact`` even though the + # AMT document correctly emits ``"fact"``. + contentType=content_type_for_memory_type(payload["type"]), + owner=validated_owner, + provenance=provenance, + acl=validated_acl, + annotations=copy.deepcopy(dict(annotations or {})), + payload=payload, + ) + except PydanticValidationError as exc: + raise MemoryEnvelopeValidationError(f"Invalid MemoryHouse envelope input: {exc}") from exc + + def unwrap(self, envelope: MemoryEnvelope | Mapping[str, Any]) -> MemoryRecordBase: + try: + # Revalidate model instances as well as mappings so this boundary + # always applies the current strict wire contract. + validated_envelope = MemoryEnvelope.model_validate(envelope) + except PydanticValidationError as exc: + raise MemoryEnvelopeValidationError(f"Invalid MemoryHouse envelope: {exc}") from exc + + payload = copy.deepcopy(validated_envelope.payload) + payload_id = payload.get("id") + payload_type = payload.get("type") + if not isinstance(payload_id, str) or not payload_id: + raise MemoryEnvelopeValidationError("MemoryHouse payload.id must be a non-empty string") + if not isinstance(payload_type, str) or not payload_type: + raise MemoryEnvelopeValidationError("MemoryHouse payload.type must be a non-empty string") + + # The global envelope ID, provenance source ID, and payload ID must all + # identify the same AMT record. None is allowed to silently override the + # others because that would make attribution and deletion unsafe. + _, envelope_memory_id = parse_envelope_id(validated_envelope.id) + if envelope_memory_id != payload_id: + raise MemoryEnvelopeIntegrityError( + f"Envelope ID memory component {envelope_memory_id!r} does not match payload.id {payload_id!r}" + ) + if validated_envelope.provenance.source_id != payload_id: + raise MemoryEnvelopeIntegrityError( + f"provenance.sourceId {validated_envelope.provenance.source_id!r} " + f"does not match payload.id {payload_id!r}" + ) + + content_memory_type = memory_type_for_content_type(validated_envelope.content_type) + if content_memory_type != payload_type: + raise MemoryEnvelopeIntegrityError( + f"Content type represents {content_memory_type!r}, but payload.type is {payload_type!r}" + ) + + try: + # Dispatch through AMT's existing discriminator so subtype-specific + # validation remains centralized in the established record models. + return MemoryRecordBase.from_doc(payload) + except (PydanticValidationError, TypeError, ValueError) as exc: + raise MemoryEnvelopeValidationError(f"Invalid AMT payload: {exc}") from exc + + +__all__ = [ + "AMT_APPLICATION_NAME", + "AMT_MEMORY_CONTENT_TYPES", + "AMTMemoryEnvelopeAdapter", + "build_envelope_id", + "content_type_for_memory_type", + "memory_type_for_content_type", + "parse_envelope_id", +] diff --git a/azure/cosmos/agent_memory/memoryhouse/exceptions.py b/azure/cosmos/agent_memory/memoryhouse/exceptions.py new file mode 100644 index 0000000..9a35cb0 --- /dev/null +++ b/azure/cosmos/agent_memory/memoryhouse/exceptions.py @@ -0,0 +1,48 @@ +"""Exceptions raised by the MemoryHouse envelope integration. + +The adapter has its own exception family because envelope validation failures +are different from AMT record validation and Cosmos persistence failures. +Callers can catch :class:`MemoryEnvelopeError` for the whole integration while +still distinguishing malformed input, unsupported formats, and integrity +violations. +""" + +from azure.cosmos.agent_memory.exceptions import AgentMemoryError + + +class MemoryEnvelopeError(AgentMemoryError): + """Base exception for MemoryHouse envelope failures.""" + + error_code = "memory_envelope" + + +class MemoryEnvelopeValidationError(MemoryEnvelopeError): + """Raised when an envelope or envelope input is structurally malformed.""" + + error_code = "memory_envelope_validation" + + +class UnsupportedMemoryContentTypeError(MemoryEnvelopeError): + """Raised when an envelope uses an unsupported content type or version.""" + + error_code = "unsupported_memory_content_type" + + +class MemoryEnvelopeIntegrityError(MemoryEnvelopeError): + """Raised when duplicated identity or type assertions disagree. + + MemoryHouse intentionally keeps the AMT payload opaque, but the envelope + still repeats the AMT ID for global addressing and the AMT type in the + content type. These checks prevent an envelope from pointing at one record + while carrying another record as its payload. + """ + + error_code = "memory_envelope_integrity" + + +__all__ = [ + "MemoryEnvelopeError", + "MemoryEnvelopeIntegrityError", + "MemoryEnvelopeValidationError", + "UnsupportedMemoryContentTypeError", +] diff --git a/azure/cosmos/agent_memory/memoryhouse/models.py b/azure/cosmos/agent_memory/memoryhouse/models.py new file mode 100644 index 0000000..c7c98d2 --- /dev/null +++ b/azure/cosmos/agent_memory/memoryhouse/models.py @@ -0,0 +1,133 @@ +"""MemoryHouse envelope wire models. + +These models describe only the standardized MemoryHouse envelope. They do not +attempt to model AMT's semantic payload because that payload remains owned by +AMT and is reconstructed through ``MemoryRecordBase.from_doc()``. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator + + +class _EnvelopeModel(BaseModel): + """Strict base model shared by every standardized envelope component.""" + + # Accept Pythonic field names when constructing models while retaining the + # camel-case aliases required by the MemoryHouse JSON wire contract. + # ``extra="forbid"`` is important here: in particular, it rejects a stored + # ``scope`` field that could drift away from ``owner.type``. + model_config = ConfigDict( + populate_by_name=True, + extra="forbid", + # Callers may pass an already-created model back through the adapter. + # Revalidating instances keeps that boundary strict even if a mutable + # model was changed after its original construction. + revalidate_instances="always", + ) + + +def _require_non_empty(value: str, *, field_name: str) -> str: + """Validate identifiers without rewriting caller-owned values.""" + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +class MemoryOwner(_EnvelopeModel): + """Entity that owns a memory and therefore defines its scope.""" + + type: Literal["user", "team", "organization"] + id: str + + @field_validator("id") + @classmethod + def _validate_id(cls, value: str) -> str: + return _require_non_empty(value, field_name="owner.id") + + @property + def scope(self) -> Literal["personal", "team", "organization"]: + """Return the human-facing scope without serializing redundant state.""" + # ``owner.type`` is the single persisted source of truth. Keeping this + # mapping as a property gives callers the requested scope vocabulary + # without introducing a second field that must be synchronized. + return { + "user": "personal", + "team": "team", + "organization": "organization", + }[self.type] + + +class MemoryACL(_EnvelopeModel): + """Principals authorized for each MemoryHouse operation.""" + + read: list[str] + write: list[str] + annotate: list[str] + forget: list[str] + + @field_validator("read", "write", "annotate", "forget") + @classmethod + def _validate_principals(cls, values: list[str], info: ValidationInfo) -> list[str]: + # Empty operation lists are valid and mean that no principal is granted + # that operation. Empty principal strings are invalid because they make + # authorization intent ambiguous. + for value in values: + _require_non_empty(value, field_name=f"acl.{info.field_name} principal") + return values + + +class MemoryProvenance(_EnvelopeModel): + """Origin and writer metadata for a MemoryHouse envelope.""" + + application: str + agent_id: str | None = Field(default=None, alias="agentId") + created_by: str = Field(alias="createdBy") + created_at: str = Field(alias="createdAt") + source_id: str = Field(alias="sourceId") + prompt_id: str | None = Field(default=None, alias="promptId") + prompt_version: str | None = Field(default=None, alias="promptVersion") + + @field_validator("application", "created_by", "created_at", "source_id") + @classmethod + def _validate_required_string(cls, value: str, info: ValidationInfo) -> str: + return _require_non_empty(value, field_name=str(info.field_name)) + + @field_validator("agent_id", "prompt_id", "prompt_version") + @classmethod + def _validate_optional_string(cls, value: str | None, info: ValidationInfo) -> str | None: + if value is None: + return None + return _require_non_empty(value, field_name=str(info.field_name)) + + +class MemoryEnvelope(_EnvelopeModel): + """Governed MemoryHouse envelope containing an opaque AMT payload.""" + + id: str + content_type: str = Field(alias="contentType") + owner: MemoryOwner + provenance: MemoryProvenance + acl: MemoryACL + annotations: dict[str, Any] + payload: dict[str, Any] + + @field_validator("id", "content_type") + @classmethod + def _validate_required_string(cls, value: str, info: Any) -> str: + return _require_non_empty(value, field_name=str(info.field_name)) + + @property + def scope(self) -> Literal["personal", "team", "organization"]: + """Expose the owner-derived scope directly on the envelope.""" + return self.owner.scope + + +__all__ = [ + "MemoryACL", + "MemoryEnvelope", + "MemoryOwner", + "MemoryProvenance", +] diff --git a/tests/unit/memoryhouse/__init__.py b/tests/unit/memoryhouse/__init__.py new file mode 100644 index 0000000..be6e2cc --- /dev/null +++ b/tests/unit/memoryhouse/__init__.py @@ -0,0 +1 @@ +"""Tests for the MemoryHouse envelope integration.""" diff --git a/tests/unit/memoryhouse/test_adapter.py b/tests/unit/memoryhouse/test_adapter.py new file mode 100644 index 0000000..41a97a3 --- /dev/null +++ b/tests/unit/memoryhouse/test_adapter.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import copy + +import pytest + +from azure.cosmos.agent_memory.memoryhouse import ( + AMT_MEMORY_CONTENT_TYPES, + AMTMemoryEnvelopeAdapter, + MemoryACL, + MemoryEnvelopeIntegrityError, + MemoryEnvelopeValidationError, + MemoryOwner, + UnsupportedMemoryContentTypeError, + build_envelope_id, + content_type_for_memory_type, + memory_type_for_content_type, + parse_envelope_id, +) +from azure.cosmos.agent_memory.models import ( + EpisodicRecord, + FactRecord, + ProceduralRecord, + ThreadSummaryRecord, + TurnRecord, + UserSummaryRecord, +) + +_HEX32 = "a" * 32 + + +def _records(): + """Build representative instances of every supported AMT record subtype.""" + return [ + TurnRecord(id="turn/1", user_id="u1", thread_id="t1", role="user", content="Hello"), + ThreadSummaryRecord( + id="summary_" + _HEX32, + user_id="u1", + thread_id="t1", + content="Summary", + prompt_id="summarize.prompty", + ), + UserSummaryRecord( + id="user_summary_" + _HEX32, + user_id="u1", + thread_id="__user_summary__", + content="User summary", + metadata={"thread_ids": ["t1"]}, + prompt_id="user_summary.prompty", + ), + FactRecord( + id="fact_" + _HEX32, + user_id="u1", + thread_id="t1", + content="User prefers concise responses.", + metadata={"category": "preference"}, + content_hash=_HEX32, + prompt_id="extract_memories.prompty", + confidence=0.9, + embedding=[0.1, 0.2], + ), + EpisodicRecord( + id="ep_" + _HEX32, + user_id="u1", + thread_id="t1", + content="The deployment succeeded.", + title="Deployment", + events=[{"sequence": 1, "description": "Deployed the service."}], + content_hash=_HEX32, + prompt_id="extract_episode.prompty", + ), + ProceduralRecord( + id="proc_u1_1", + user_id="u1", + thread_id="t1", + content="Validate before deploying.", + name="Safe deployment", + summary="Validate changes before deployment.", + retrieval_text="deployment validation", + procedure_kind="behavioral_policy", + prompt_id="extract_procedure.prompty", + ), + ] + + +def _acl() -> MemoryACL: + return MemoryACL( + read=["user:u1"], + write=["user:u1"], + annotate=["service:enricher"], + forget=["user:u1"], + ) + + +@pytest.mark.parametrize( + "store_id,memory_id", + [ + ("default", "fact_1"), + ("west/us", "turn/1"), + ("store with spaces", "memory % value"), + ], +) +def test_envelope_id_round_trip(store_id: str, memory_id: str): + envelope_id = build_envelope_id(store_id, memory_id) + assert parse_envelope_id(envelope_id) == (store_id, memory_id) + + +@pytest.mark.parametrize("envelope_id", ["", "other/a/b", "amt/a", "amt/a/b/c", "amt/a/%ZZ", "amt/a/b c"]) +def test_parse_envelope_id_rejects_invalid_values(envelope_id: str): + with pytest.raises(MemoryEnvelopeValidationError): + parse_envelope_id(envelope_id) + + +@pytest.mark.parametrize("memory_type", sorted(AMT_MEMORY_CONTENT_TYPES)) +def test_content_type_round_trip(memory_type: str): + content_type = content_type_for_memory_type(memory_type) + assert memory_type_for_content_type(content_type) == memory_type + media_type, version = content_type.split(";") + assert memory_type_for_content_type(f" {media_type} ; {version} ") == memory_type + + +def test_content_type_rejects_unknown_type_and_version(): + with pytest.raises(UnsupportedMemoryContentTypeError): + content_type_for_memory_type("unknown") + with pytest.raises(UnsupportedMemoryContentTypeError): + memory_type_for_content_type("application/vnd.microsoft.amt.fact+json;version=2") + + +@pytest.mark.parametrize("record", _records(), ids=lambda record: record.to_doc()["type"]) +def test_wrap_unwrap_is_lossless_for_every_record_type(record): + # This equality is the central compatibility guarantee: governance may be + # added around a record, but AMT sees exactly the same document after + # unwrapping. + adapter = AMTMemoryEnvelopeAdapter() + annotations = {"classification": {"domain": "payments"}} + envelope = adapter.wrap( + record, + store_id="default", + owner=MemoryOwner(type="user", id="u1"), + acl=_acl(), + created_by="agent-1", + annotations=annotations, + ) + + assert envelope.scope == "personal" + assert envelope.annotations == annotations + assert set(envelope.annotations) == {"classification"} + assert envelope.payload == record.to_doc() + assert envelope.content_type == AMT_MEMORY_CONTENT_TYPES[record.to_doc()["type"]] + assert adapter.unwrap(envelope).to_doc() == record.to_doc() + + +def test_wrap_deep_copies_payload_and_annotations(): + # Both directions contain nested mutable values. The adapter must isolate + # them so callers cannot accidentally mutate source records or input + # annotations through the returned envelope. + adapter = AMTMemoryEnvelopeAdapter() + record = _records()[3] + annotations = {"classification": {"keywords": ["preference"]}} + expected_payload = copy.deepcopy(record.to_doc()) + + envelope = adapter.wrap( + record, + store_id="default", + owner={"type": "team", "id": "payments"}, + acl=_acl(), + created_by="agent-1", + annotations=annotations, + ) + envelope.payload["content"] = "changed" + envelope.annotations["classification"]["keywords"].append("changed") + + assert record.to_doc() == expected_payload + assert annotations == {"classification": {"keywords": ["preference"]}} + assert envelope.scope == "team" + + +def test_unwrap_rejects_content_type_payload_mismatch(): + adapter = AMTMemoryEnvelopeAdapter() + envelope = adapter.wrap( + _records()[3], + store_id="default", + owner={"type": "user", "id": "u1"}, + acl=_acl(), + created_by="agent-1", + ) + doc = envelope.model_dump(mode="json", by_alias=True) + doc["contentType"] = AMT_MEMORY_CONTENT_TYPES["episodic"] + + with pytest.raises(MemoryEnvelopeIntegrityError, match="payload.type"): + adapter.unwrap(doc) + + +def test_unwrap_rejects_identity_mismatch(): + adapter = AMTMemoryEnvelopeAdapter() + envelope = adapter.wrap( + _records()[0], + store_id="default", + owner={"type": "organization", "id": "contoso"}, + acl=_acl(), + created_by="agent-1", + ) + doc = envelope.model_dump(mode="json", by_alias=True) + doc["provenance"]["sourceId"] = "different" + + with pytest.raises(MemoryEnvelopeIntegrityError, match="sourceId"): + adapter.unwrap(doc) diff --git a/tests/unit/memoryhouse/test_models.py b/tests/unit/memoryhouse/test_models.py new file mode 100644 index 0000000..946a2e4 --- /dev/null +++ b/tests/unit/memoryhouse/test_models.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import pydantic +import pytest + +from azure.cosmos.agent_memory.memoryhouse import MemoryACL, MemoryEnvelope, MemoryOwner, MemoryProvenance + + +def _envelope_kwargs() -> dict: + """Return one complete wire-format envelope for focused model tests.""" + return { + "id": "amt/default/turn-1", + "contentType": "application/vnd.microsoft.amt.turn+json;version=1", + "owner": {"type": "user", "id": "u1"}, + "provenance": { + "application": "agent-memory-toolkit", + "createdBy": "agent-1", + "createdAt": "2026-08-20T20:00:00+00:00", + "sourceId": "turn-1", + }, + "acl": { + "read": ["user:u1"], + "write": ["user:u1"], + "annotate": [], + "forget": ["user:u1"], + }, + "annotations": {}, + "payload": {"id": "turn-1", "type": "turn"}, + } + + +@pytest.mark.parametrize( + ("owner_type", "scope"), + [ + ("user", "personal"), + ("team", "team"), + ("organization", "organization"), + ], +) +def test_owner_type_defines_scope(owner_type: str, scope: str): + owner = MemoryOwner(type=owner_type, id="scope-id") + assert owner.scope == scope + + +def test_scope_is_not_serialized(): + # Scope is intentionally derived from owner.type and must never appear as a + # second persisted field. + envelope = MemoryEnvelope(**_envelope_kwargs()) + assert envelope.scope == "personal" + assert "scope" not in envelope.model_dump(mode="json", by_alias=True) + assert "scope" not in envelope.owner.model_dump(mode="json", by_alias=True) + + +def test_serialized_scope_is_rejected(): + # Strict envelope validation prevents callers from creating contradictory + # values such as owner.type="team" with scope="personal". + kwargs = _envelope_kwargs() + kwargs["scope"] = "personal" + with pytest.raises(pydantic.ValidationError, match="scope"): + MemoryEnvelope(**kwargs) + + +@pytest.mark.parametrize("model", [MemoryOwner, MemoryACL, MemoryProvenance, MemoryEnvelope]) +def test_standardized_models_reject_unknown_fields(model): + if model is MemoryOwner: + kwargs = {"type": "user", "id": "u1"} + elif model is MemoryACL: + kwargs = {"read": [], "write": [], "annotate": [], "forget": []} + elif model is MemoryProvenance: + kwargs = { + "application": "amt", + "createdBy": "writer", + "createdAt": "now", + "sourceId": "source", + } + else: + kwargs = _envelope_kwargs() + kwargs["unexpected"] = True + with pytest.raises(pydantic.ValidationError, match="unexpected"): + model(**kwargs) + + +def test_acl_rejects_empty_principal(): + with pytest.raises(pydantic.ValidationError, match="non-empty"): + MemoryACL(read=[""], write=[], annotate=[], forget=[]) + + +def test_envelope_serializes_wire_aliases(): + doc = MemoryEnvelope(**_envelope_kwargs()).model_dump(mode="json", by_alias=True) + assert "contentType" in doc + assert doc["provenance"]["createdBy"] == "agent-1" + assert "content_type" not in doc