diff --git a/AGENTS.md b/AGENTS.md index fd9359d..1ef054d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,11 +49,22 @@ OnlyOffice → `bin/onlyoffice/import-contact.go`; reading contacts → `bin/con Never `contacts/…`, never `brainwrite.go` (that is `bin/brain/import-contact.go`). Same rule for `var/` and `etc/` subdirs: `var/{subject}/…`, `etc/{subject}/…`. +**Standing rules:** +- **Go only, no Python** — new tooling is Go (shebang mains); Python never + returns to the pipeline. +- **Libs named by domain** — `internal/{domain}` / `pkg/{domain}` mirror their + `bin/{subject}`; files inside are concrete artifacts (`leaf.go`, `yaml.go`). + No output-type namespaces (`mdleaves`-style) — no `mdroot`/`mdcontent` creep. +- **docs/ = truth only** — what works or what is open; plans/proofs go to + Gitea issues, then get deleted here. +- **Non-goals (standing):** no behavior change to graph/search semantics; + no rework of the pluggable reasoner / CGO toolchain (D18/D21). + bin/ self-describing tools (singular, `{subject}/{verb}-{object}.go`) bin/brain/ import-contact.go import-git.go search.go serve.go index.go add.go get.go stats.go eval.go watch.go bin/contact/ list.go (read/normalize csv/vcf/mab → stdout/file) bin/onlyoffice/ import-contact.go (reconcile contacts into OO CRM) -bin/chats/ sync.go import.go facts.go apply.go; libs in internal/chats +bin/chat/ sync.go import.go facts.go apply.go refresh-session.go; lib: internal/chat bin/mail/ sync.go import.go convert-mbox.go ocr.go (lib: internal/mailsync) bin/markdown/ split-leaf.go (H2 leaf split; lib: internal/markdown) bin/jsonl/ stats.go (DuckDB quantiles / JSONL count; gcc CGO, not Zig) @@ -63,7 +74,7 @@ bin/reasoner/ bench.go (D18 CPU OpenAI tool-call bench) bin/facts/ extract.go audit.go audit-db.go prove-crm.go bin/shell/ complete.go (flaggy completions dump, D23) pkg/ public reusable Go (cli flaggy D23; repo; contact; duckdb; httpapi) — no 2dph deps -internal/ private 2dph Go (brain, chats, facts D16, gitlog, websearch, reasoner, mailsync, mailconv, corpuswatch, markdown) +internal/ private 2dph Go by domain (brain, chat, facts D16, gitlog, websearch, reasoner, mailsync, mailconv, corpuswatch, markdown) bin/cgo/ zig zcc zc++ (CGO via zig cc, not gcc) bin/stack/ start start-assistant stop status (compose + PicoClaw agent) bin/docker-entrypoint container entrypoint (api: serve|search|watch|index|add|mail-sync) @@ -149,7 +160,9 @@ duckdb-go (`pkg/duckdb`, `skills/duckdb/SKILL.md`), not Ladybug. 4. **Curasoft, edelweiss — no files, no mentions.** Remove all traces if found. 5. **Check git history before push.** If any commit contains leaks, rewrite history (rebase + force push) AND delete affected GitHub releases/tags. -6. **`docs/chat-import-plan.md`** — reference Gitea issue, never embed secrets. +6. **Plans live in Gitea issues, not docs/.** docs/ holds only what is true + and current (design, runbook, roadmap). Historical plans/proofs are + archived as comments on their Gitea issue, then deleted from the repo. ## Communication diff --git a/README.md b/README.md index f1f39f4..d13e7d5 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ domain area it acts on: | `bin/mail` | `sync.go` / `import.go` / `ocr.go` | mail ETL (Gmail/OO/M365) | | `bin/web` | `search.go` | SearXNG second source | | `bin/git` | `import.go` | commit history leafs | -| `bin/chats` | `sync.go` / `import.go` / `apply.go` | conversations | +| `bin/chat` | `sync.go` / `import.go` / `apply.go` | conversations | | `bin/stack` | `start` / `status` / `stop` | compose dispatcher | Go methods are executable (`go run` shebang); a few are thin bash launchers diff --git a/bin/chat b/bin/chat deleted file mode 100755 index 4260cea..0000000 --- a/bin/chat +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# bin/chats - sync, import, index, facts, apply for Telegram/WhatsApp/LinkedIn. -# Builds the chats binary on first run / when source changes, then execs it. -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -BIN="$ROOT/var/bin/chats" -SRC="$ROOT/bin/chats" - -mkdir -p "$ROOT/var/bin" - -need_build=0 -if [ ! -x "$BIN" ]; then - need_build=1 -else - while IFS= read -r -d '' f; do - if [ "$f" -nt "$BIN" ]; then - need_build=1 - break - fi - done < <(find "$SRC" -name '*.go' -print0 2>/dev/null) -fi - -if [ "$need_build" -eq 1 ]; then - echo "Building chats..." >&2 - (cd "$SRC" && go build -o "$BIN" .) || exit 1 -fi - -exec "$BIN" "$@" diff --git a/bin/chats/apply.go b/bin/chat/apply.go similarity index 52% rename from bin/chats/apply.go rename to bin/chat/apply.go index b03bd88..e48cbcd 100755 --- a/bin/chats/apply.go +++ b/bin/chat/apply.go @@ -1,9 +1,9 @@ //usr/bin/env go run -tags=chats_apply "$0" "$@"; exit //go:build chats_apply // -// bin/chats/apply.go - push extracted chat facts to OnlyOffice CRM. +// bin/chat/apply.go - push extracted chat facts to OnlyOffice CRM. // -// ./bin/chats/apply.go [--dry-run] +// ./bin/chat/apply.go [--dry-run] // // NOTE: never run `gofmt -w` on this file — it breaks the shebang. package main @@ -11,9 +11,9 @@ package main import ( "os" - "github.com/eSlider/2dph/internal/chats" + "github.com/eSlider/2dph/internal/chat" ) func main() { - os.Exit(chats.RunApply(os.Args[1:])) + os.Exit(chat.RunApply(os.Args[1:])) } diff --git a/bin/chats/doc.go b/bin/chat/doc.go similarity index 67% rename from bin/chats/doc.go rename to bin/chat/doc.go index 4a8cb8d..ea1c096 100644 --- a/bin/chats/doc.go +++ b/bin/chat/doc.go @@ -1,4 +1,4 @@ // Commands in this directory are shebang mains (sync.go, import.go, facts.go, -// apply.go), each behind an exclusive build tag so `go build ./bin/chats` -// does not see two mains. Shared code lives in internal/chats. +// apply.go), each behind an exclusive build tag so `go build ./bin/chat` +// does not see two mains. Shared code lives in internal/chat. package main diff --git a/bin/chats/facts.go b/bin/chat/facts.go similarity index 59% rename from bin/chats/facts.go rename to bin/chat/facts.go index 3b50580..db3b98b 100755 --- a/bin/chats/facts.go +++ b/bin/chat/facts.go @@ -1,9 +1,9 @@ //usr/bin/env go run -tags=chats_facts "$0" "$@"; exit //go:build chats_facts // -// bin/chats/facts.go - extract phone/email/linkedin facts from JSONL. +// bin/chat/facts.go - extract phone/email/linkedin facts from JSONL. // -// ./bin/chats/facts.go +// ./bin/chat/facts.go // // Writes var/chats/facts/. Does not index the brain. // NOTE: never run `gofmt -w` on this file — it breaks the shebang. @@ -12,9 +12,9 @@ package main import ( "os" - "github.com/eSlider/2dph/internal/chats" + "github.com/eSlider/2dph/internal/chat" ) func main() { - os.Exit(chats.RunFacts(os.Args[1:])) + os.Exit(chat.RunFacts(os.Args[1:])) } diff --git a/bin/chats/import.go b/bin/chat/import.go similarity index 62% rename from bin/chats/import.go rename to bin/chat/import.go index de020ef..6cbfa7d 100755 --- a/bin/chats/import.go +++ b/bin/chat/import.go @@ -1,9 +1,9 @@ //usr/bin/env go run -tags=chats_import "$0" "$@"; exit //go:build chats_import // -// bin/chats/import.go - JSONL → markdown under var/chats/md/. +// bin/chat/import.go - JSONL → markdown under var/chats/md/. // -// ./bin/chats/import.go +// ./bin/chat/import.go // // Conversion only. Brain ingest is bin/brain/index.go, not this command. // NOTE: never run `gofmt -w` on this file — it breaks the shebang. @@ -12,9 +12,9 @@ package main import ( "os" - "github.com/eSlider/2dph/internal/chats" + "github.com/eSlider/2dph/internal/chat" ) func main() { - os.Exit(chats.RunImport(os.Args[1:])) + os.Exit(chat.RunImport(os.Args[1:])) } diff --git a/bin/chat/refresh-session.go b/bin/chat/refresh-session.go new file mode 100644 index 0000000..b579ebc --- /dev/null +++ b/bin/chat/refresh-session.go @@ -0,0 +1,48 @@ +//usr/bin/env go run "$0" "$@"; exit +// +// bin/chat/refresh-session.go - refresh LinkedIn MCP session from webtop CDP. +// +// ./bin/chat/refresh-session.go [--cdp URL] [--root DIR] +// [--container work-webtop] [--profile thorium-profile] +// +// Reads live cookies out of the running Thorium browser via CDP +// (Network.getAllCookies), copies the browser profile, rewrites cookies.json + +// source-state.json. Run before every linkedin sync. +package main + +import ( + "fmt" + "os" + + "github.com/eSlider/2dph/internal/chat" + cliparse "github.com/eSlider/2dph/pkg/cli" +) + +func main() { + os.Exit(run(os.Args[1:])) +} + +func run(args []string) int { + var ( + cdp string + root string + container string + profile string + ) + p := cliparse.New("chat-refresh-session") + p.Description = "refresh LinkedIn session from webtop CDP (cookies.json + source-state.json)" + p.String(&cdp, "", "cdp", "CDP endpoint (default http://127.0.0.1:9222)") + p.String(&root, "", "root", "portable profile root (default /var/tmp/liprofile)") + p.String(&container, "", "container", "webtop container (default work-webtop)") + p.String(&profile, "", "profile", "browser profile dir (default thorium-profile)") + if err := cliparse.Parse(p, args); err != nil { + return cliparse.Fail(err) + } + if err := chat.RefreshLinkedInSession(chat.RefreshLinkedInSessionOpts{ + CDP: cdp, Root: root, Container: container, Profile: profile, + }); err != nil { + fmt.Fprintf(os.Stderr, "chat-refresh-session: %v\n", err) + return 1 + } + return 0 +} diff --git a/bin/chats/sync.go b/bin/chat/sync.go similarity index 56% rename from bin/chats/sync.go rename to bin/chat/sync.go index 299116f..69b24b1 100755 --- a/bin/chats/sync.go +++ b/bin/chat/sync.go @@ -1,10 +1,10 @@ //usr/bin/env go run -tags=chats_sync "$0" "$@"; exit //go:build chats_sync // -// bin/chats/sync.go - download chat messages to var/chats//. +// bin/chat/sync.go - download chat messages to var/chats//. // -// ./bin/chats/sync.go telegram [--limit N] [--phone PHONE] -// ./bin/chats/sync.go linkedin [--limit N] [--refresh] +// ./bin/chat/sync.go telegram [--limit N] [--phone PHONE] +// ./bin/chat/sync.go linkedin [--limit N] [--refresh] // // NOTE: never run `gofmt -w` on this file — it breaks the shebang. package main @@ -13,26 +13,26 @@ import ( "fmt" "os" - "github.com/eSlider/2dph/internal/chats" + "github.com/eSlider/2dph/internal/chat" ) func main() { if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, `usage: bin/chats/sync.go telegram|linkedin [flags]`) + fmt.Fprintln(os.Stderr, `usage: bin/chat/sync.go telegram|linkedin [flags]`) os.Exit(2) } platform := os.Args[1] args := os.Args[2:] switch platform { case "telegram": - os.Exit(chats.RunSyncTelegram(args)) + os.Exit(chat.RunSyncTelegram(args)) case "linkedin": - os.Exit(chats.RunSyncLinkedIn(args)) + os.Exit(chat.RunSyncLinkedIn(args)) case "whatsapp": fmt.Fprintln(os.Stderr, "chats: WhatsApp sync is out of v1") os.Exit(1) case "help", "-h", "--help": - fmt.Fprintln(os.Stderr, `usage: bin/chats/sync.go telegram|linkedin [flags] + fmt.Fprintln(os.Stderr, `usage: bin/chat/sync.go telegram|linkedin [flags] WhatsApp sync is out of v1.`) return default: diff --git a/bin/chats/refresh-linkedin-session b/bin/chats/refresh-linkedin-session deleted file mode 100755 index c62fec4..0000000 --- a/bin/chats/refresh-linkedin-session +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python3 -"""chats/refresh-linkedin-session - refresh LinkedIn MCP session from webtop CDP. - - bin/chats/refresh-linkedin-session [--cdp URL] [--root DIR] -""" - -Reads the current LinkedIn cookies out of the running Thorium browser in the -work-webtop container via CDP (Network.getAllCookies), copies the live browser -profile onto the source profile directory, and rewrites the portable -cookies.json + source-state.json that mcp-server-linkedin requires. - -Usage: - refresh-linkedin-session [--cdp http://127.0.0.1:9222] [--root /var/tmp/liprofile] - [--container work-webtop] [--profile thorium-profile] - -After the headless driver uses a copied profile, LinkedIn rotates the session -in that copy, so this must run before every sync. -""" - -import asyncio -import json -import os -import shutil -import subprocess -import sys -import tempfile -import urllib.request - -import websockets - - -def cdp_tab(ws_json): - for t in ws_json: - if t.get("webSocketDebuggerUrl"): - return t["webSocketDebuggerUrl"] - return None - - -async def get_cookies(ws_url): - async with websockets.connect(ws_url, max_size=50_000_000) as ws: - await ws.send(json.dumps({"id": 1, "method": "Network.getAllCookies", "params": {}})) - resp = await ws.recv() - return json.loads(resp).get("result", {}).get("cookies", []) - - -def write_source_state(root, profile_dir): - # Reuse the linkedin-mcp-server session_state module to write a valid - # source-state.json (same schema the daemon reads). - try: - from linkedin_mcp_server.session_state import canonical, write_source_state - - write_source_state(canonical(__import__("pathlib").Path(profile_dir))) - return - except Exception: - pass - # Fallback: minimal schema-compatible state. - import uuid - - state = { - "version": 1, - "source_runtime_id": "linux-amd64-host", - "login_generation": str(uuid.uuid4()), - "created_at": None, - "profile_path": profile_dir, - "cookies_path": os.path.join(root, "cookies.json"), - } - from datetime import datetime, timezone - - state["created_at"] = datetime.now(timezone.utc).isoformat() - with open(os.path.join(root, "source-state.json"), "w") as f: - json.dump(state, f, indent=2) - - -def main(): - args = sys.argv[1:] - cdp = "http://127.0.0.1:9222" - root = "/var/tmp/liprofile" - container = "work-webtop" - cprofile = "thorium-profile" - for i in range(0, len(args), 2): - k = args[i] - v = args[i + 1] if i + 1 < len(args) else "" - if k == "--cdp": - cdp = v - elif k == "--root": - root = v - elif k == "--container": - container = v - elif k == "--profile": - cprofile = v - - profile_dir = os.path.join(root, "profile") - os.makedirs(profile_dir, exist_ok=True) - - # 1. Clear stale daemon/browser locks so the server can claim the profile. - for lock in ("profile-claim.lock", "profile.lock", "daemon.lock", "lease.lock"): - p = os.path.join(root, lock) - if os.path.exists(p): - os.remove(p) - for name in os.listdir(profile_dir): - if name.startswith("Singleton"): - os.remove(os.path.join(profile_dir, name)) - for name in os.listdir(root): - if name.startswith("invalid-state-"): - shutil.rmtree(os.path.join(root, name), ignore_errors=True) - - # 1. Copy the live browser profile (cookies DB + Local State) so the - # session the driver launches carries the current login. - subprocess.run( - ["docker", "cp", f"{container}:/config/{cprofile}/Default", os.path.join(profile_dir, "Default")], - check=True, capture_output=True, - ) - subprocess.run( - ["docker", "cp", f"{container}:/config/{cprofile}/Local State", os.path.join(profile_dir, "Local State")], - check=True, capture_output=True, - ) - for lock in ("SingletonLock", "SingletonCookie", "SingletonSocket"): - p = os.path.join(profile_dir, lock) - if os.path.exists(p): - os.remove(p) - - # 2. Pull the live cookies out of the running browser. - with urllib.request.urlopen(f"{cdp}/json", timeout=5) as r: - tabs = json.loads(r.read()) - ws_url = cdp_tab(tabs) - if not ws_url: - sys.stderr.write("refresh-linkedin-session: no CDP tab\n") - sys.exit(1) - cookies = asyncio.run(get_cookies(ws_url)) - - li = [c for c in cookies if "linkedin" in c.get("domain", "")] - out = [] - for c in li: - domain = c.get("domain", "") - if domain in (".www.linkedin.com", "www.linkedin.com"): - domain = ".linkedin.com" - out.append({ - "name": c["name"], - "value": c["value"].strip('"'), - "domain": domain, - "path": c.get("path", "/"), - "expires": c.get("expires", -1), - "httpOnly": c.get("httpOnly", False), - "secure": c.get("secure", False), - "sameSite": c.get("sameSite", "None"), - }) - with open(os.path.join(root, "cookies.json"), "w") as f: - json.dump(out, f, indent=2) - - write_source_state(root, profile_dir) - sys.stderr.write(f"refresh-linkedin-session: {len(out)} cookies, profile refreshed\n") - - -if __name__ == "__main__": - main() diff --git a/bin/shell/complete.go b/bin/shell/complete.go index b464feb..5270c86 100755 --- a/bin/shell/complete.go +++ b/bin/shell/complete.go @@ -16,7 +16,7 @@ import ( mailsync "github.com/eSlider/2dph/internal/mailsync" "github.com/eSlider/2dph/internal/brain/rank" - "github.com/eSlider/2dph/internal/chats" + "github.com/eSlider/2dph/internal/chat" "github.com/eSlider/2dph/pkg/cli" "github.com/eSlider/2dph/internal/gitlog" "github.com/eSlider/2dph/internal/markdown" @@ -42,10 +42,10 @@ func tools() []cli.Tool { {Path: "bin/reasoner/bench.go", Name: "reasoner-bench", New: reasoner.Parser}, {Path: "bin/mail/ocr.go", Name: "mail-ocr", New: ocr.Parser}, {Path: "bin/mail/sync.go", Name: "mail-sync", New: mailsync.Parser}, - {Path: "bin/chats/mailsync.go", Name: "chats-sync", New: chats.SyncParser}, - {Path: "bin/chats/import.go", Name: "chats-import", New: chats.ImportParser}, - {Path: "bin/chats/facts.go", Name: "chats-facts", New: chats.FactsParser}, - {Path: "bin/chats/apply.go", Name: "chats-apply", New: chats.ApplyParser}, + {Path: "bin/chat/mailsync.go", Name: "chat-sync", New: chat.SyncParser}, + {Path: "bin/chat/import.go", Name: "chat-import", New: chat.ImportParser}, + {Path: "bin/chat/facts.go", Name: "chat-facts", New: chat.FactsParser}, + {Path: "bin/chat/apply.go", Name: "chat-apply", New: chat.ApplyParser}, } } diff --git a/docs/README.md b/docs/README.md index 2bb79cf..41e3a12 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,36 +5,45 @@ related: - docs/runbook.md - docs/design.md - PLAN.md - - docs/roadmap.md --- -# 2dph docs (Diataxis) +# 2dph docs Evidence-first knowledge graph. Facts need proof or they are `(not confirmed)`. +**Docs policy:** only what is true and current — what works, or what is +open (tracked as Gitea issues). Plans and historical proofs live as Gitea +issue comments, not here. + | Type | Doc | |------|-----| -| tutorial / howto | [runbook](runbook.md) — run anywhere (uv, Go, Docker) | -| explanation | [design](design.md) — two roots, deduction, D17/D20/D18 | -| explanation | [roadmap](roadmap.md) — gap to v1 (epic #16) | -| howto | [picoclaw](picoclaw.md) — MCP agent (`bin/stack/start-assistant`) | -| howto | [reasoner](reasoner.md) — CPU bake-off (D18) | -| reference | [PLAN.md](../PLAN.md) — decisions D1–D24 | +| howto (run) | [runbook](runbook.md) — build, config, serve/search/index | +| explanation | [design](design.md) — two roots, deduction, versioning, read path | +| reference | [PLAN.md](../PLAN.md) — decisions D1–D25 | +| howto (brain ops) | [brain/rebuild](brain/rebuild.md) — parallel write, resume | +| howto (facts) | [facts/audit-recipes](facts/audit-recipes.md) — audit recipes | +| howto (agent) | [picoclaw](picoclaw.md) — MCP gateway (`bin/stack/start-assistant`) | +| howto (reasoner) | [reasoner](reasoner.md) — CPU bench (D18) | | testing | [test/README.md](../test/README.md) — system / stress / integration tiers | | config | `etc/{searxng,picoclaw}` — operator config (FHS) | -Decisions the public face must name: **D3** SearXNG compose, **D6** Go service / -Python write sidecar, **D14** `bin/{subject}/{method}.go`, **D15** Gitea origin, -**D17** assertion gate (facts → info → web), **D18** pluggable reasoner. +Historical: roadmap (gap to v1, closed epic #16) kept for context; +load baseline 2026-08-11 → Gitea #58. -Search: `bin/brain/search.go "query"` (HTTP: `bin/brain/serve.go` — -`/health` `/search` `/get` `/stats` `/audit` `/ingest`). `--hop N` walks -`FROM_FILE` → Commit → Person from each hit (max 3). Rebuild writes -File edges ([#17](https://git.produktor.io/eSlider/2dph/issues/17)). +## Quick start -Work board: [Gitea issues](https://git.produktor.io/eSlider/2dph/issues) -([epic #16](https://git.produktor.io/eSlider/2dph/issues/16)). -PRs and CI: GitHub [`eSlider/2dph`](https://github.com/eSlider/2dph). +```bash +bin/brain/search.go "query" # deduction: facts → info → web +bin/brain/serve.go # HTTP + OpenAPI + MCP (:8630) +bin/brain/index.go --rebuild # bulk rebuild (Zig CGO) +``` + +MCP surface: `search` / `get` / `audit`. Full OpenAPI: `GET /openapi.json`. -Published docs live here and match live commands. +## Work board + +[Gitea issues](https://git.produktor.io/eSlider/2dph/issues): +epic #66 (import → brain + CRM, priority), #62 (cash sprint), +#64 (conversations v2), milestone [v2](https://git.produktor.io/eSlider/2dph/milestone/13). +PRs and CI: GitHub [`eSlider/2dph`](https://github.com/eSlider/2dph). diff --git a/docs/brain-rebuild.md b/docs/brain/rebuild.md similarity index 100% rename from docs/brain-rebuild.md rename to docs/brain/rebuild.md diff --git a/docs/chat-import-plan.md b/docs/chat-import-plan.md deleted file mode 100644 index 3f574b1..0000000 --- a/docs/chat-import-plan.md +++ /dev/null @@ -1,25 +0,0 @@ -# Chat Import Pipeline - -Plan: https://git.produktor.io/eSlider/brain-chats-import/issues/1 - -## Env vars (set in shell, never committed) - -``` -TELEGRAM_MCP_DIR -TELEGRAM_API_ID / TELEGRAM_API_HASH / TELEGRAM_PHONE -TELEGRAM_SESSION_STRING -ONLYOFFICE_URL / ONLYOFFICE_USER / ONLYOFFICE_PASS -OO_CLI (default: $HOME/go/bin/oo) -``` - -## Quick reference - -``` -./bin/chats/sync.go telegram --limit 100 -./bin/chats/import.go -./bin/chats/facts.go -./bin/chats/apply.go --dry-run -``` - -JSONL → markdown only. Brain ingest is `bin/brain/index.go --with-chats` -(default `var/chats/md`). WhatsApp sync is out of v1. diff --git a/docs/design.md b/docs/design.md index 4012e67..bba088f 100644 --- a/docs/design.md +++ b/docs/design.md @@ -36,7 +36,7 @@ bin/brain/search.go "question" `--hop N` walks `Leaf-[:FROM_FILE]->File-[:HAS_VERSION]->Commit-[:AUTHORED]->Person` from each hit (1=File, 2=Commit, 3=Person). Rebuild writes FROM_FILE; -git import writes HAS_VERSION/AUTHORED ([#17](https://git.produktor.io/eSlider/2dph/issues/17)). +`bin/brain/import-git.go` writes HAS_VERSION/AUTHORED ([#17](https://git.produktor.io/eSlider/2dph/issues/17)). Go CLIs parse with **flaggy** via `pkg/cli` (D23). Flags may appear after positionals (`search q --hop 1`). Completions: @@ -63,10 +63,10 @@ Content leafs: `sha256`, `observed_at`, `source_rev`, `confidence`. Stale = a file changed on disk (git HEAD/mtime) after its last observed `source_rev`. `File-[:HAS_VERSION]->Commit-[:AUTHORED]->Person` records the history of every content leaf. Commit records come from `bin/brain/import-git.go` (go-git, no git -binary); conversion prints leafs, brain write is `bin/brain/index.go`. +binary); it upserts commit leafs into the brain directly, `--dry-run` previews. -`bin/facts/audit stale` flags leafs whose observed revision is behind the -corpus HEAD. +`bin/facts/audit-db` gates evidence over the store (`root=facts`); leafs whose +observed revision is behind the corpus HEAD are flagged there. Fact **interval of truth** (D24 / OQ5): leaf props `valid_from` / `valid_to` (YYYY-MM-DD, inclusive; empty end = open; both empty = legacy @@ -95,15 +95,14 @@ Conflicting pairings (≥2 yes vs ≥2 no) stay hypothesis until They do not exec Python. Control questions for recall@5 live in `internal/brain/rank` so CI can test the table without libladybug. Bulk index/write is Go-only: `bin/brain/index.go` + `bin/brain/add.go` -(both Zig CGO, `docker compose --profile index`). Python `bin/kb/index` is -deprecated, kept only for A/B comparison. +(both Zig CGO, `docker compose --profile index`). ## Agent API (D20) `bin/brain/serve.go` exposes the same `pkg/httpapi.Ops` table as OpenAPI (`GET /openapi.json`) and MCP (`POST /mcp` JSON-RPC `tools/list` + -`tools/call`). Tool names match paths: `search`, `get`, `stats`, `audit`, -`ingest` (add a leaf; omit body for the CLI hint). +`tools/call`). **MCP surface is 3 tools** — `search`, `get`, `audit` (the +detective lever). `stats` and `ingest` remain OpenAPI HTTP paths only. Agents should use these endpoints instead of shebang CLIs. ## Reasoner (D18) diff --git a/docs/audit-recipes.md b/docs/facts/audit-recipes.md similarity index 98% rename from docs/audit-recipes.md rename to docs/facts/audit-recipes.md index 14b63da..eb54ad7 100644 --- a/docs/audit-recipes.md +++ b/docs/facts/audit-recipes.md @@ -81,7 +81,7 @@ flags facts that lost their two-source form. Every claim's `source` must resolve to an artifact the brain actually holds: - mail: `var/mail/md` (M365 sync → `bin/mail/sync.go` → `brain/index --with-mail`) -- telegram/n chat: `var/chats` (`bin/chats/sync.go n|linkedin`) +- telegram/n chat: `var/chats` (`bin/chat/sync.go n|linkedin`) - corpus: `cv/`, `projects/knowledge-mesh-seed.yaml` ```bash diff --git a/docs/load-test-summary-2026-08-11.md b/docs/load-test-summary-2026-08-11.md deleted file mode 100644 index 784d609..0000000 --- a/docs/load-test-summary-2026-08-11.md +++ /dev/null @@ -1,118 +0,0 @@ -# Brain Load Test Summary - -**Date**: 2026-08-11 -**Project**: 2dph (deductionphile) -**Target**: LadybugDB-embedded knowledge graph brain - -## Test Suite - -Four independent load tests were written and executed in `qa/`: - -| Test | Purpose | Key Finding | -|------|---------|-------------| -| `load_test_search.py` | FTS, vector, hybrid search latency | FTS: 2.8ms, Vector: 1.9ms, Hybrid: 3.3ms | -| `load_test_graph.py` | Cypher hop traversal (1-hop, 2-hop, 3-hop) | 1-hop: 2.8ms, 2-hop: 4.7ms, 3-hop: 6.6ms | -| `load_test_queries.py` | Query pattern diversity (9 patterns) | All patterns under 25ms | -| `load_test_bulk.py` | Bulk insert throughput (leafs/sec) | 251 leafs/sec (with index drop/recreate) | - -## Results - -### 1. Search Performance (`load_test_search.py`, 10 iterations) - -| Mode | Avg Latency (ms) | Description | -|------|-----------------|-------------| -| FTS (BM25) | **2.8 ms** | Pure keyword search | -| Vector (HNSW cosine) | **1.9 ms** | Embedding similarity search | -| Hybrid (RRF merge) | **3.3 ms** | FTS + vector fusion | - -**Observation**: All modes under 5ms. Hybrid is ~1.8x slower than individual modes due to RRF overhead, but still well under 25ms per query. - -### 2. Graph Traversal (`load_test_graph.py`, 10 iterations) - -| Pattern | Avg Latency (ms) | Description | -|---------|-----------------|-------------| -| 1-hop (Leaf -FROM_FILE-> File) | **2.8 ms** | Simple edge traversal | -| 2-hop (Leaf -> File -> Commit) | **4.7 ms** | Two-hop path with mix node types | -| 3-hop (facts -from_file-> File -> HAS_VERSION-> Commit -AUTHORED-> Person) | **6.6 ms** | Three-hop path with root filter | -| Degree centrality (avg children per file) | **4.2 ms** | Aggregation query | - -**Observation**: Graph queries are very fast (<10ms even for 3 hops) on the knowledge graph. - -### 3. Query Pattern Diversity (`load_test_queries.py`, 10 iterations) - -| Query Pattern | Avg Latency (ms) | -|---------------|-----------------| -| fact_source (docker, root=facts) | 5.3 | -| info_docker (docker, root=info) | 3.4 | -| info_k8s (kubernetes, root=info) | 3.2 | -| repo_2dph (search term, repo=eSlider/2dph) | 3.6 | -| facts_no_root (search, root=facts) | 4.1 | -| hybrid_container (container, hybrid search) | 3.9 | -| hybrid_service (service, hybrid search) | 3.7 | -| multi_obs (observability, multi-word) | 4.3 | -| multi_container (container orchestration, multi-word) | 4.7 | - -**Observation**: All 9 query patterns complete in under 25ms. The system correctly handles root-filtered and repo-filtered searches. - -### 4. Bulk Insert (`load_test_bulk.py`, 30 leafs, indexes dropped before insert) - -| Metric | Value | -|--------|-------| -| Total time for 30 leafs | 0.12s | -| Throughput | **251 leafs/sec** | - -**Critical observation (corrected 2026-08-12)**: LadybugDB **0.19** must **not** -`DROP INDEX` for FTS/VECTOR and recreate. DROP leaves ghost catalog tables -(`_0_Leaf_vec_UPPER`, `0_id_docs`); CREATE then fails with "already exists in -catalog" while `SHOW_INDEXES` omits the index — HNSW looks dead until -`var/kb.lbug` is deleted. Upsert while indexes exist keeps HNSW queryable. -Fresh indexes: delete the DB file and `bin/kb/index --rebuild`. See -`kblib.create_fts_and_vector` / `ensure_indexes`. - -## Critical Assessment - Evidence Rule Working - -The most important finding: **the evidence-based audit correctly enforces the two-source rule for facts**. - -- `bin/facts/audit db` runs against `var/kb.lbug` and asserts each `root=facts` leaf has: - - A `source` field containing " x " (indicating two independent sources, e.g., "docker ps x compose:docker-compose.yml") - - A non-empty `loc` (evidence pointer) - - `confidence='confirmed'` - -- **Before cleanup**: Database had 50 test facts with `source="load-test"` (single source) → audit correctly flagged all as failing the 2-source rule -- **After cleanup (12 facts from extract)**: Audit passes (`ok: true, problems: []`) because the 12 facts have proper 2-source evidence: - - 11 facts: `source="docker ps x compose:..."` or `source="docker ps x compose:..."` - - 1 fact: `source="ssh config x docs(README.md, PLAN.md, AGENTS.md)"` - -This validates the core design principle from PLAN.md (D8/D11): **a fact needs ≥2 independent sources or it is `(not confirmed)`**. - -## Database State (After Cleanup) - -| Metric | Value | -|--------|-------| -| Total leaves | 89 (47 info + 12 facts) | -| Facts (root=facts) | 12, all with 2-source evidence | -| Info (root=info) | 47 (from markdown corpus) | -| Audit result | `ok: true, problems: []` | - -## Files (historical, `qa/` at the time) - -- `load_test_search.py` - Search latency test (FT/Vector/Hybrid) -- `load_test_graph.py` - Graph traversal test (1-hop, 2-hop, 3-hop) -- `load_test_queries.py` - Query pattern diversity test (9 patterns) -- `load_test_bulk.py` - Bulk insert throughput test -- `load_test_summary.md` - This summary - -## Verdict - -The brain performs well within design parameters: - -- **Search/retrieval latency**: sub-25ms across all modes -- **Graph traversal**: under 10ms even for 3-hop paths -- **Bulk insertion**: ~250 leafs/sec (with proper index management) -- **Evidence enforcement**: The two-source audit correctly validates facts, confirming the detective method works as designed (`facts` root = strong assertions, `info` root = weak claims) - -The system is ready for production use with the understanding that: -1. Bulk inserts must drop/recreate indexes to avoid corruption -2. Facts are only stored when backed by >=2 independent sources (enforced by audit) -3. The info root holds the narrative corpus (28K+ markdown-derived leafs) -4. Facts root holds confirmed assertions with evidence links \ No newline at end of file diff --git a/docs/mail-import-rewrite-plan.md b/docs/mail-import-rewrite-plan.md deleted file mode 100644 index 7befa8b..0000000 --- a/docs/mail-import-rewrite-plan.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -type: plan -status: draft -related: - - docs/runbook.md - - docs/design.md - - PLAN.md ---- - -# Plan: rewrite mail import with a Go MIME email library + type-handler attachments - -## Goal - -Replace the current extension-only mail → markdown conversion with a proper -MIME email parser that: - -1. Reads the **raw email** (headers + body + multipart) instead of relying on - the pre-parsed `message.json` shape. -2. Preserves the **original email date** as a first-class, sortable leaf field - (prerequisite for time-based deduction — see background below). -3. Determines **attachment type by MIME type + extension** and routes each - attachment through a **type handler** (PDF/OCR, image/OCR, text, office, - structured, unknown). - -## Background / why - -- The brain cannot do date-based reconstruction/deduction today: mail leafs - carry **no date**. `msgDate()` (`internal/brain/corpus.go:222`) reads - `message.json receivedDate/receivedAt` **only** as a `--since` sync filter; - `WriteCorpus` (`corpus.go:240`) never sets `LeafInput.ValidFrom/ValidTo`. - `LeafInput` and the Leaf schema already have `valid_from/valid_to` (D24). -- Attachment handling is by **file extension** only - (`internal/mailconv/mailconv.go:69` `ConvertAttachment`), so a `.pdf` named - `.txt` or an inline `image/*` part is mis-handled. MIME type + extension is - more reliable. - -## Library selection - -Primary: **`github.com/jhillyerd/enmime/v2`** (v2.4.1, 2026). - -- Purpose-built MIME email parser (Inbucket project), production quality. -- Parses headers, multipart/alternative, inline + attachment parts. -- Per-part: `ContentType`, `FileName`, `Disposition`, decoded `Content`. -- Header decoding (RFC 2047), charset handling. - -Alternative: **`github.com/emersion/go-message`** (streaming RFC 5322/2045-2047, -charset via `.../charset`). Use if streaming memory profile is preferred. - -Decision: **enmime/v2**. Rationale: highest-level email API, built-in attachment -decoding, active maintenance (v2.4.x 2026). - -## Architecture - -``` -raw email (EML / .eml / oo mail body) - │ enmime.ReadEnvelope(r) - ▼ -Envelope{ From, To, Date, Subject, Text, HTML, Attachments: []Part } - │ normalize + StripHTML(HTML)→Markdown - ▼ -mailconv.Message{ id, source, folder, subject, from, to, date, body_md, attachments } - │ per attachment: TypeHandlerRegistry.Route(MIME, ext) → handler - ▼ -message.md (frontmatter: date, from, to, subject, ...) + attachments/*.md - │ WriteCorpus(LeafInput{ ValidFrom: date, ... }) - ▼ -brain leaf (info, dated) → date-sortable search/deduction -``` - -## Type-handler registry (MIME + extension) - -`internal/mailconv/handlers.go` — a deterministic map keyed by -`(mimeType, extension)` → handler func `(path, name, mime) (md string, err)`. - -| Priority | MIME type | Ext | Handler | -|----------|-----------|-----|---------| -| 1 | `application/pdf` | `.pdf` | PDF: `pdftotext -layout`; if textless → `pdftoppm`+`tesseract eng+deu` (reuse `internal/ocr`) | -| 2 | `image/*` | `.png .jpg .jpeg .gif .webp .tiff` | image OCR (`internal/ocr.ImageFile`) when `--ocr` | -| 3 | `text/plain` | `.txt .log .csv` | pass-through (or CSV→markdown table) | -| 4 | `text/html` | `.html .htm` | `StripHTML` → markdown | -| 5 | `text/calendar` | `.ics` | `golang-ical` → summary events (already a dep) | -| 6 | `application/json`/`xml` | `.json .xml` | structured → code block | -| 7 | `application/vnd.openxmlformats-...` | `.docx .xlsx .pptx` | (optional) unzip+text extraction; else metadata-only | -| 8 | `application/msword`, `application/vnd.ms-excel` | `.doc .xls` | metadata-only (v1) | -| 9 | default / unknown | any | metadata-only: `name · MIME · size` | - -Rules: -- Route on **MIME first**, extension second (MIME is authoritative). -- Handler failures must not fail the import → emit ``. -- Registry is data-driven (table) so handlers can be added without touching - the loop. - -## Date preservation - -- Parse `Envelope.Date` (enmime gives a parsed `time.Time`) → store in - `mailconv.Message.Date`. -- Emit `date:` in `message.md` frontmatter (already present, keep). -- Set `LeafInput.ValidFrom = date`, `ValidTo = ""` in `WriteCorpus` so the - leaf is date-queryable. (Schema already supports it; no migration.) - -## File/layout changes - -- `go.mod`: add `github.com/jhillyerd/enmime/v2`. -- `internal/mailconv/mailconv.go`: rework `Message` to hold MIME-derived data; - `FromRaw` parse raw email via enmime, keep producing `message.md`. -- `internal/mailconv/handlers.go` (new): type-handler registry. -- `internal/mailconv/handlers_test.go` (new): fixtures per MIME/ext. -- `internal/brain/corpus.go`: `WriteCorpus` sets `ValidFrom` from `CorpusLeaf.Date`. -- `internal/brain/search.go` + MCP `search`: date filter (`--as-of` → apply to - info/mail too) + `sort by valid_from asc|desc` + return date in `get`. - -## Verification - -- Fixtures: `.eml` files with multipart/alternative + attachments of each - MIME/ext class; assert correct body + handler routing. -- Golden test: a known `.eml` → expected `message.md` + `attachments/*.md`. -- Re-import sample (e.g. `bin/mail/import.go --from-raw var/mail --ocr`) → - leafs have `valid_from`. -- `bin/brain/search.go "defacto" --as-of ` returns dated mail; `get` - shows the date; `--sort date asc` orders correctly. -- Memory: confirm enmime streamed parsing keeps RSS bounded on the ~18k corpus. - -## Rollout - -1. Add enmime dep + registry + unit tests (offline fixtures). -2. Rework `mailconv.FromRaw` to parse via enmime (keep output shape). -3. Wire `ValidFrom` in `WriteCorpus`. -4. Extend `search`/`get` for date filter + sort. -5. Re-import + rebuild the ~18k mail corpus. -6. Verify search/sort/deduction; document in `docs/`. - -## Out of scope (follow-ups) - -- Importing the `@defacto.de` mailbox (separate source) so defacto-era emails - become searchable by date. -- Time-based deduction algorithms on top of the now-dated corpus. diff --git a/docs/refactor-v1.0.md b/docs/refactor-v1.0.md deleted file mode 100644 index a5615be..0000000 --- a/docs/refactor-v1.0.md +++ /dev/null @@ -1,135 +0,0 @@ -# Refactor v1.0 — plan (Musk method) - -Goal: cut clutter so each command, dir, service and MCP tool earns its place. -Evidence-first (Sherlock) applied to the tool surface itself. - -Status: **in progress 2026-08-21** — P1–P4 done (cleanup, compose verify, MCP 5→3). -Open: **P5 tests taxonomy** (`qa/`→`test/{integration,system}`), **P6 FHS `etc/`** -(`deploy/`→`etc/{subject}/`), **P7 documentation** (project description + usage), -P8 (CI + PR). Epic [v1.0 #65](https://git.produktor.io/eSlider/2dph/issues/65), -milestone [v1.0](https://git.produktor.io/eSlider/2dph/milestone/16). -Parallel to cash-sprint [epic #62](https://git.produktor.io/eSlider/2dph/issues/62) — not closed. - -## Measure first - -| Parameter | Before (2026-08-21) | After | Target | -|-----------|--------------------:|------:|-------:| -| bin/ tool files (`.go` + executables) | 72 | ~67 | ~34 | -| bin/ dirs | 19 | 18 | ~15 | -| internal/ dirs | 13 | 13 | ~11 | -| compose services | 9 (3 default + 6 profile) | 9 | keep (already lean) | -| MCP tools | 5 | **3** | 3 | -| root config dirs (`deploy/`, `qa/`) | 2 | 0 (folded into `etc/`, `test/`) | 0 | - -## A/B (delete / keep) - -### Delete (verified 2026-08-21, each is deprecated/dup) -- `bin/kb/{add,index,search,watch}` — deprecated wrappers → real Go in `bin/brain/*` (PLAN.md D6). Delete dir. -- `bin/tools/web-search` — only test fixtures; move to testdata, drop dir. `bin/tools/__pycache__` — junk. -- `bin/serve.go`, `bin/fulfill-assoc.go`, `bin/doc.go` root shims — fold into `bin/brain/` or `bin/facts/`, delete root copies. - -### Keep (verified not dup) -- `bin/chat` — launcher bash that builds+execs `bin/chats/` (not a dup of `chats/`). Keep. -- `bin/markdown/split-leaf.go` — real tool (H2→leaf), wired into `bin/shell/complete.go`. Keep. -- `bin/postgres/query.go` — active Go read-only wrapper over `bin/db/psql-yq`, documented. Keep. -- `bin/ci/semver.go` — **active**: CI Release job computes semver. Keep. -- `bin/fulfill-assoc.go` — active (recent commit c6d27c9, #52/#55). Keep. - -### Keep (one clear owner each) -- `bin/brain/*` (11) — core read/write/search/serve. -- `bin/cgo/*` (zig/zcc/zc++) — toolchain. -- `bin/chats/*` — conversations. -- `bin/onlyoffice/import-contact.go`, `bin/brain/import-contact.go`, `bin/contact/list.go` — contacts (subject = target). -- `bin/db/*` — psql-yq, ssh-tunnel. -- `bin/facts/*` — audit/extract/crm. -- `bin/brain/import-git.go` — history. -- `bin/mail/*` — sync/import/ocr. -- `bin/jsonl/stats.go` — duckdb. -- `bin/reasoner/bench.go` — CPU bake-off. -- `bin/stack/*` — compose dispatcher. -- `bin/shell/complete.go` — flaggy completion. -- `bin/web/search.go` — SearXNG. - -## compose (verified 2026-08-21) - -9 services: **3 default** (`brain`, `brain-watch`, `mail-sync`) + **6 profile** -(`index`, `searxng`, `reasoner`, `picoclaw`, `ocr-paddle`, plus `brain-mcp` under -picoclaw). `reasoner-ollama` / `picoclaw-home` are named **volumes** (needed), -not stub services. Already lean — no merge needed. Verify P2 only that -default set stays minimal. - -## MCP 5 → 3 - -`search`, `get`, `audit` are the detective lever (#15). Fold `stats` into `search` -(scores/quantiles block) and `ingest` into `get`? No — `ingest` is a write. Decision: -expose only `search` / `get` / `audit`; move `stats` behind `search --stats`, -`ingest` behind `POST /ingest` (OpenAPI, not MCP). Tools/list shows 3. - -## P5 — tests taxonomy: `qa/` → `test/{integration,system}` - -Current `qa/` is a grab-bag: `system_perf*.go`, `stress/`, `load_test_summary.md`. -Rework into a clear taxonomy (D-test): - -- `test/system/` — offline-gated system tests (no live brain in CI): recall eval, - perf gates, source-gate checks. Corresponds to current `qa/system_perf*`. -- `test/integration/` — tests that need a live dependency (OnlyOffice CRM, SearXNG, - Ladybug DB, mail) — opt-in via build tag / env, not run by default `go test ./...`. -- `test/stress/` — load/stress scenarios (from `qa/stress/`). -- `test/README.md` — how to run each tier (`go test ./test/system/...`, - `go test -tags=integration ./test/integration/...`). - -CI keeps running only the offline `system` tier by default. - -## P6 — FHS config: `deploy/` → `etc/{subject}/` - -Move operator-edited config out of `deploy/` into `etc/{subject}/`, one per tool: - -- `deploy/searxng/{settings.yml,limiter.toml}` → `etc/searxng/{settings.yml,limiter.toml}` -- `deploy/picoclaw/{config.json,mcp.json.example}` → `etc/picoclaw/…` -- `compose.yaml` stays at root (compose convention) but env templates → - `etc/{brain,mail-sync}/.env.example`; examples of `db-profiles.yml` / `search.env` - → `etc/{brain,postgres}/…example`. -- runtime output stays in `var/` (gitignored) — already done. - -Compose paths and any `deploy/` references updated. `docker compose config -q` stays clean. - -## mapstructure/v2 (D34) — verified 2026-08-21, not needed - -Config loaders already use **`yaml.v3` struct tags** (`bin/facts/extract.go` → -`Services map[string]any`, `bin/fulfill-assoc.go` → typed `Org`/`Orgs`, -`bin/postgres/query.go` → thin bash wrapper over `bin/db/psql-yq`). There is no -hand-rolled map→struct to replace. Applying mapstructure would be a regression. -Decision: **keep yaml.v3 struct tags**; do not add mapstructure. - -## P7 — documentation rework - -Project must read as "what is this, how to use it" from a cold start: - -- **README.md** — rewrite: what 2dph is, one-line pitch, quick start (build, serve, - search), CLI tour, links to docs. Not an exhaustive tool list. -- **docs/runbook.md** — operational: build, config (etc/ + `~/.config/brain`), - serve/search/watch/index, compose profiles, common tasks, troubleshooting. -- **docs/design.md** — architecture + decisions (already good; link from README). -- **docs/refactor-v1.0.md** — this plan. -- Verify: a fresh reader can go README → runbook → serve → search in ~15 min. - -## Phases - -| Phase | Work | Exit | -|-------|------|------| -| P1 | Snapshot bin/ + compose; verify delete list; delete `bin/kb/`, fold root shims; relocate web-search fixtures → testdata | no `bin/kb`, no root shims, fixtures in testdata | -| P2 | (verified) compose 9 services, 3 default + 6 profile — already lean | `docker compose config -q` clean | -| P3 | (cancelled after verify) `markdown`/`postgres`/`ci` are active & documented — keep | bin/ dirs = 19, all live | -| P4 | MCP 5→3 (stats/ingest out), OpenAPI paths stay, testdata reloc | tools/list = 3 | -| P5 | tests taxonomy `qa/` → `test/{system,integration,stress}` + test/README | `go test ./test/system/...` green; no root `qa/` | -| P6 | FHS config `deploy/` → `etc/{subject}/`; env templates; compose refs updated | no root `deploy/`; `docker compose config -q` clean | -| P7 | docs rework: README (what/how), runbook, design link, test/README | cold-start → serve → search in ~15 min | -| P8 | CI green (system tier), PLAN updated, PR | CI green | - -## Non-goals - -- No Python reintroduction. -- No behavior change to graph/search semantics. -- No rework of PLUGABLE reasoner / CGO toolchain. -- `bin/jsonl/stats.go` (DuckDB) stays — it is a tool, not the test taxonomy `qa/`. -- Not blocking cash-sprint #62. diff --git a/docs/roadmap.md b/docs/roadmap.md index 09961c1..4f00d3b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ --- type: explanation -status: current +status: historical (v1 closed, epic #16; current board: Gitea #66 + milestone v2) related: - PLAN.md - docs/design.md diff --git a/internal/chats/apply.go b/internal/chat/apply.go similarity index 99% rename from internal/chats/apply.go rename to internal/chat/apply.go index ab16bab..2c3c008 100644 --- a/internal/chats/apply.go +++ b/internal/chat/apply.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "bytes" diff --git a/internal/chats/chats_test.go b/internal/chat/chats_test.go similarity index 99% rename from internal/chats/chats_test.go rename to internal/chat/chats_test.go index e9c150a..98eade2 100644 --- a/internal/chats/chats_test.go +++ b/internal/chat/chats_test.go @@ -1,9 +1,9 @@ -// System tests for bin/chats. +// System tests for bin/chat. // // These are integration tests using real data and real Telegram API (when // credentials are available). They follow the TDD workflow pattern: // sync → import → facts → verify. -package chats +package chat import ( "encoding/json" diff --git a/internal/chats/cli.go b/internal/chat/cli.go similarity index 99% rename from internal/chats/cli.go rename to internal/chat/cli.go index 358b713..6fd45af 100644 --- a/internal/chats/cli.go +++ b/internal/chat/cli.go @@ -1,4 +1,4 @@ -package chats +package chat import ( cliparse "github.com/eSlider/2dph/pkg/cli" diff --git a/internal/chats/facts.go b/internal/chat/facts.go similarity index 99% rename from internal/chats/facts.go rename to internal/chat/facts.go index 81fbbeb..c8e2b22 100644 --- a/internal/chats/facts.go +++ b/internal/chat/facts.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "bufio" diff --git a/internal/chats/import.go b/internal/chat/import.go similarity index 99% rename from internal/chats/import.go rename to internal/chat/import.go index 1a8d94e..1126950 100644 --- a/internal/chats/import.go +++ b/internal/chat/import.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "bufio" diff --git a/internal/chats/linkedin.go b/internal/chat/linkedin.go similarity index 99% rename from internal/chats/linkedin.go rename to internal/chat/linkedin.go index f768e8d..b6296cc 100644 --- a/internal/chats/linkedin.go +++ b/internal/chat/linkedin.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "bufio" diff --git a/internal/chat/linkedin_session.go b/internal/chat/linkedin_session.go new file mode 100644 index 0000000..21a346f --- /dev/null +++ b/internal/chat/linkedin_session.go @@ -0,0 +1,227 @@ +package chat + +// LinkedIn session refresh: pull live cookies out of the webtop Thorium +// browser via CDP and rebuild the portable profile the LinkedIn MCP server +// (and chats/sync) consume. Go port of the former python helper. + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "golang.org/x/net/websocket" +) + +// RefreshLinkedInSessionOpts selects the CDP endpoint, the portable profile +// root and the container/profile to copy from. +type RefreshLinkedInSessionOpts struct { + CDP string // default http://127.0.0.1:9222 + Root string // default /var/tmp/liprofile + Container string // default work-webtop + Profile string // default thorium-profile +} + +type cdpCookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Expires float64 `json:"expires"` + HTTPOnly bool `json:"httpOnly"` + Secure bool `json:"secure"` + SameSite string `json:"sameSite"` +} + +type cdpTab struct { + WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` +} + +// RefreshLinkedInSession refreshes cookies.json + source-state.json under +// opts.Root from the live browser profile in the webtop container. +func RefreshLinkedInSession(opts RefreshLinkedInSessionOpts) error { + if opts.CDP == "" { + opts.CDP = "http://127.0.0.1:9222" + } + if opts.Root == "" { + opts.Root = "/var/tmp/liprofile" + } + if opts.Container == "" { + opts.Container = "work-webtop" + } + if opts.Profile == "" { + opts.Profile = "thorium-profile" + } + profileDir := filepath.Join(opts.Root, "profile") + if err := os.MkdirAll(profileDir, 0o755); err != nil { + return err + } + + // 1. Clear stale daemon/browser locks so the server can claim the profile. + for _, lock := range []string{"profile-claim.lock", "profile.lock", "daemon.lock", "lease.lock"} { + _ = os.Remove(filepath.Join(opts.Root, lock)) + } + if ents, err := os.ReadDir(profileDir); err == nil { + for _, e := range ents { + if strings.HasPrefix(e.Name(), "Singleton") { + _ = os.Remove(filepath.Join(profileDir, e.Name())) + } + } + } + if ents, err := os.ReadDir(opts.Root); err == nil { + for _, e := range ents { + if strings.HasPrefix(e.Name(), "invalid-state-") { + _ = os.RemoveAll(filepath.Join(opts.Root, e.Name())) + } + } + } + + // 2. Copy the live browser profile (cookies DB + Local State). + for _, src := range []string{"Default", "Local State"} { + dst := filepath.Join(profileDir, src) + cmd := exec.Command("docker", "cp", opts.Container+":/config/"+opts.Profile+"/"+src, dst) + cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("docker cp %s: %w", src, err) + } + } + for _, lock := range []string{"SingletonLock", "SingletonCookie", "SingletonSocket"} { + _ = os.Remove(filepath.Join(profileDir, lock)) + } + + // 3. Pull live cookies from the running browser over CDP. + tabs, err := cdpTabs(opts.CDP) + if err != nil { + return err + } + var wsURL string + for _, t := range tabs { + if t.WebSocketDebuggerURL != "" { + wsURL = t.WebSocketDebuggerURL + break + } + } + if wsURL == "" { + return fmt.Errorf("no CDP tab with webSocketDebuggerUrl") + } + cookies, err := cdpAllCookies(wsURL) + if err != nil { + return err + } + + // 4. Filter linkedin domains, normalize, write cookies.json. + out := make([]map[string]any, 0, len(cookies)) + for _, c := range cookies { + domain := c.Domain + if !strings.Contains(domain, "linkedin") { + continue + } + switch domain { + case ".www.linkedin.com", "www.linkedin.com": + domain = ".linkedin.com" + } + expires := c.Expires + if expires == 0 { + expires = -1 + } + sameSite := c.SameSite + if sameSite == "" { + sameSite = "None" + } + out = append(out, map[string]any{ + "name": c.Name, + "value": strings.Trim(c.Value, `"`), + "domain": domain, + "path": orDefault(c.Path, "/"), + "expires": expires, + "httpOnly": c.HTTPOnly, + "secure": c.Secure, + "sameSite": sameSite, + }) + } + if err := writeJSONFile(filepath.Join(opts.Root, "cookies.json"), out); err != nil { + return err + } + if err := writeSourceState(opts.Root, profileDir); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "refresh-session: %d linkedin cookies, profile refreshed\n", len(out)) + return nil +} + +func cdpTabs(cdp string) ([]cdpTab, error) { + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(strings.TrimRight(cdp, "/") + "/json") + if err != nil { + return nil, err + } + defer resp.Body.Close() + var tabs []cdpTab + return tabs, json.NewDecoder(resp.Body).Decode(&tabs) +} + +func cdpAllCookies(wsURL string) ([]cdpCookie, error) { + ws, err := websocket.Dial(wsURL, "", "http://127.0.0.1") + if err != nil { + return nil, fmt.Errorf("cdp dial: %w", err) + } + defer ws.Close() + req := map[string]any{"id": 1, "method": "Network.getAllCookies", "params": map[string]any{}} + if err := websocket.JSON.Send(ws, req); err != nil { + return nil, err + } + var resp struct { + ID int `json:"id"` + Result struct { + Cookies []cdpCookie `json:"cookies"` + } `json:"result"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + if err := websocket.JSON.Receive(ws, &resp); err != nil { + return nil, err + } + if resp.Error != nil { + return nil, fmt.Errorf("cdp: %s", resp.Error.Message) + } + return resp.Result.Cookies, nil +} + +// writeSourceState writes a schema-compatible source-state.json for the +// linkedin mcp daemon (minimal fallback form). +func writeSourceState(root, profileDir string) error { + id, err := uuid.NewRandom() + if err != nil { + return err + } + state := map[string]any{ + "version": 1, + "source_runtime_id": "linux-amd64-host", + "login_generation": id.String(), + "created_at": time.Now().UTC().Format(time.RFC3339), + "profile_path": profileDir, + "cookies_path": filepath.Join(root, "cookies.json"), + } + return writeJSONFile(filepath.Join(root, "source-state.json"), state) +} + +func writeJSONFile(path string, v any) error { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func orDefault(s, def string) string { + if s == "" { + return def + } + return s +} diff --git a/internal/chats/linkedin_test.go b/internal/chat/linkedin_test.go similarity index 99% rename from internal/chats/linkedin_test.go rename to internal/chat/linkedin_test.go index cdf4a94..f567124 100644 --- a/internal/chats/linkedin_test.go +++ b/internal/chat/linkedin_test.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "errors" diff --git a/internal/chats/mcpclient.go b/internal/chat/mcpclient.go similarity index 99% rename from internal/chats/mcpclient.go rename to internal/chat/mcpclient.go index 306de76..a68e71c 100644 --- a/internal/chats/mcpclient.go +++ b/internal/chat/mcpclient.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "bufio" diff --git a/internal/chats/paths.go b/internal/chat/paths.go similarity index 97% rename from internal/chats/paths.go rename to internal/chat/paths.go index a74fcab..c9d8b23 100644 --- a/internal/chats/paths.go +++ b/internal/chat/paths.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "os" diff --git a/internal/chats/source.go b/internal/chat/source.go similarity index 98% rename from internal/chats/source.go rename to internal/chat/source.go index 677a0ce..8d45022 100644 --- a/internal/chats/source.go +++ b/internal/chat/source.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "context" diff --git a/internal/chats/sync_linkedin.go b/internal/chat/sync_linkedin.go similarity index 79% rename from internal/chats/sync_linkedin.go rename to internal/chat/sync_linkedin.go index a139c08..d7b185d 100644 --- a/internal/chats/sync_linkedin.go +++ b/internal/chat/sync_linkedin.go @@ -1,10 +1,9 @@ -package chats +package chat import ( "context" "fmt" "os" - "os/exec" "path/filepath" "time" @@ -76,23 +75,10 @@ func RunSyncLinkedIn(args []string) int { } // refreshLinkedInSession re-syncs the LinkedIn source session from the live -// webtop browser via the vendored refresh-linkedin-session helper. +// webtop browser (CDP cookies + profile copy) — in-process, no helper script. func refreshLinkedInSession(userDataDir string) int { - exe, err := os.Executable() - if err != nil { - fmt.Fprintf(os.Stderr, "chats: resolve executable: %v\n", err) - return 1 - } - helper := filepath.Join(filepath.Dir(exe), "refresh-linkedin-session") - if _, err := os.Stat(helper); err != nil { - // Fall back to the source tree helper next to this command file. - helper = "bin/chats/refresh-linkedin-session" - } root := filepath.Dir(userDataDir) - cmd := exec.Command(helper, "--root", root) - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { + if err := RefreshLinkedInSession(RefreshLinkedInSessionOpts{Root: root}); err != nil { fmt.Fprintf(os.Stderr, "chats: linkedin session refresh: %v\n", err) return 1 } diff --git a/internal/chats/sync_telegram.go b/internal/chat/sync_telegram.go similarity index 99% rename from internal/chats/sync_telegram.go rename to internal/chat/sync_telegram.go index da4d9dc..0742efd 100644 --- a/internal/chats/sync_telegram.go +++ b/internal/chat/sync_telegram.go @@ -1,4 +1,4 @@ -package chats +package chat import ( "context" diff --git a/internal/chats/testdata/linkedin_conversation.json b/internal/chat/testdata/linkedin_conversation.json similarity index 100% rename from internal/chats/testdata/linkedin_conversation.json rename to internal/chat/testdata/linkedin_conversation.json diff --git a/internal/chats/testdata/linkedin_inbox.json b/internal/chat/testdata/linkedin_inbox.json similarity index 100% rename from internal/chats/testdata/linkedin_inbox.json rename to internal/chat/testdata/linkedin_inbox.json diff --git a/test/README.md b/test/README.md index 0ac5cef..707f8a6 100644 --- a/test/README.md +++ b/test/README.md @@ -10,4 +10,4 @@ Tiers (refactor v1.0 P5): CI runs the offline **system** tier by default. -Historical load report: [docs/load-test-summary-2026-08-11.md](../docs/load-test-summary-2026-08-11.md). +Historical load baseline 2026-08-11: Gitea issue #58 (comment archive).