diff --git a/Documentation/reference/prologue.md b/Documentation/reference/prologue.md index 7a800f4a..f0d07c00 100644 --- a/Documentation/reference/prologue.md +++ b/Documentation/reference/prologue.md @@ -45,11 +45,13 @@ Reads Prologue capture (`.jsonl`) files and interprets them into a Screenplay. D |---|---| | `--file ` | File to write the generated Screenplay to. Defaults to `.play` in the current directory. | | `--prologue-id ` | The Prologue the captures belong to. Defaults from `cratis-prologue.json` when one is present. | +| `--no-llm` | Force heuristics-only interpretation. Never creates a chat client or sends capture evidence to a model, regardless of local or global configuration. | ```bash cratis prologue interpret cratis prologue interpret ./captures cratis prologue interpret ./captures --file MySystem.play +cratis prologue interpret ./captures --no-llm ``` On success it prints a panel with the written path, the derived system name, and the module/feature/slice counts — and the natural next step is `cratis run` to boot the Screenplay in a local [Stage](run.md) sandbox. @@ -58,9 +60,18 @@ On success it prints a panel with the written path, the derived system name, and The language model is resolved in this order: -1. The `llm` section of a found `cratis-prologue.json`, when it is enabled there. -2. The `llm` section of `~/.cratis/config.json`, written by [`cratis llm use`](llm.md) — `anthropic`, `openai`, or `local` (OpenAI-compatible). -3. Neither configured — interpretation runs with heuristics only and an info line points at `cratis llm use`. +1. `--no-llm` disables refinement, overriding every configuration setting. +2. An explicit `llm.enabled` in a found `cratis-prologue.json`: `true` uses the local provider; `false` disables refinement **without falling back to the global provider**. +3. When the local `llm.enabled` setting is absent, the `llm` section of `~/.cratis/config.json`, written by [`cratis llm use`](llm.md) — `anthropic`, `openai`, or `local` (OpenAI-compatible). +4. Neither configured — interpretation runs with heuristics only. + +With `--no-llm` or local `llm.enabled: false`, no chat client is created and no capture evidence leaves the machine for a model. Use `--no-llm` for scripts that must never contact a model. + +Before creating the interpreter session, the command prints the effective provider kind, model id, endpoint host, and setting source to **stderr**. This notice is also printed with `-y/--yes`, JSON output, `--quiet`, or no terminal; a globally configured model is not silently used. Only the endpoint host is shown, never credentials, paths, or query strings. When refinement is disabled, the notice says `none` and heuristics-only mode. + +For Anthropic, a custom endpoint takes precedence over `ANTHROPIC_BASE_URL`; an empty endpoint or the default Ollama URL (`http://llm:11434`) is treated as unset. When neither a custom endpoint nor the environment variable is set, the public Anthropic API is used. The resolved endpoint is fixed before the notice and passed to the client. The OpenAI provider uses its public API; other providers use their configured endpoint. An enabled provider's effective endpoint must be an absolute HTTP or HTTPS URL with a host (for example, `http://127.0.0.1:11434`); an invalid endpoint causes a validation error before any capture evidence is sent. + +Table and plain results include the provider notice. JSON results include an `llm` object with `used`, `kind`, `model`, `endpointHost`, and `source`, including when no model is used. `source` is `local file`, `global config`, `--no-llm`, or `none`; a disabled model has `used: false`, `kind: "none"`, and empty model and endpoint host values. `used` indicates that refinement was enabled for the run, not that a provider returned a usable refinement. Quiet text output remains the written file path; the notice still appears on stderr. When the language model is genuinely uncertain about a decision that materially changes the model, it asks questions — one at a time, each with its background context, a list of choices, and always an "Other" entry for typing your own answer. Questions are only asked in an interactive terminal; non-interactive runs (CI, piped output, `-y/--yes`) never ask and finalize with the model's best effort. @@ -70,6 +81,7 @@ When the language model is genuinely uncertain about a decision that materially |---|---| | `start` without an interactive terminal, or with `--yes` | Validation error — the wizard needs a terminal. | | `interpret` finds no capture (`.jsonl`) files in the folder | Not-found error with a hint to run the extractor with JSON output. | +| Enabled model's effective endpoint is invalid | Validation error before creating a chat client or sending capture evidence. | | Interpretation fails | Server error carrying the session's error message. | ## Running Prologue without the CLI diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/given/an_interpret_command.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/given/an_interpret_command.cs new file mode 100644 index 00000000..b559b2e0 --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/given/an_interpret_command.cs @@ -0,0 +1,102 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Cratis.Prologue.Configuration; +using Cratis.Prologue.Contracts; +using Cratis.Prologue.Interpretation; +using Microsoft.Extensions.AI; +using Spectre.Console; + +namespace Cratis.Cli.for_InterpretPrologueCommand.given; + +public class an_interpret_command : Specification +{ + protected string _folder; + protected InterpretPrologueSettings _settings; + protected InterpretPrologueCommand _command; + protected IChatClientFactory _chatClients; + protected IChatClient _client; + protected LlmConfiguration? _global; + protected int _globalLoads; + protected string _noticeBeforeClient; + protected string _noticeBeforeRequest; + protected string _output; + protected string _notice; + protected int _exitCode; + protected JsonElement _llm; + + void Establish() + { + _folder = Directory.CreateTempSubdirectory("cli-241-").FullName; + File.WriteAllText(Path.Combine(_folder, "cratis-prologue.json"), "{}"); + File.WriteAllText(Path.Combine(_folder, "http.jsonl"), CaptureFiles.Serialize(new CapturedEntry( + Guid.NewGuid(), + DateTimeOffset.UtcNow, + SourceKind.Http, + new HttpCommandObserved("POST", "/api/orders", 201, string.Empty)))); + _settings = new InterpretPrologueSettings + { + Path = _folder, + File = Path.Combine(_folder, "System.play"), + Output = OutputFormats.Json, + Yes = true + }; + _global = new() { Kind = "openai", Model = "global-model", ApiKey = "global-secret" }; + _noticeBeforeClient = string.Empty; + _noticeBeforeRequest = string.Empty; + _chatClients = Substitute.For(); + _client = Substitute.For(); + _command = new InterpretPrologueCommand(_chatClients, () => + { + _globalLoads++; + return _global; + }); + } + + protected void Configure([StringSyntax(StringSyntaxAttribute.Json)] string json) => File.WriteAllText(Path.Combine(_folder, "cratis-prologue.json"), json); + + protected async Task Interpret() + { + var previousOutput = Console.Out; + var previousError = Console.Error; + var previousConsole = AnsiConsole.Console; + await using var output = new StringWriter(); + await using var error = new StringWriter(); + _chatClients.CreateFor(Arg.Any()).Returns(_ => + { + _noticeBeforeClient = error.ToString(); + return _client; + }); + _client.GetResponseAsync(Arg.Any>(), Arg.Any(), Arg.Any()).Returns(_ => + { + _noticeBeforeRequest = error.ToString(); + return new ChatResponse(new ChatMessage(ChatRole.Assistant, "{}")); + }); + try + { + Console.SetOut(output); + Console.SetError(error); + AnsiConsole.Console = AnsiConsole.Create(new AnsiConsoleSettings { Out = new AnsiConsoleOutput(output), Ansi = AnsiSupport.No, Interactive = InteractionSupport.No }); + AnsiConsole.Console.Profile.Width = 300; + _exitCode = await _command.ExecuteAsync(new CommandContext([], Substitute.For(), "interpret", null), _settings, CancellationToken.None); + } + finally + { + Console.SetOut(previousOutput); + Console.SetError(previousError); + AnsiConsole.Console = previousConsole; + } + + _output = output.ToString(); + _notice = error.ToString(); + if (_exitCode == ExitCodes.Success && _settings.ResolveOutputFormat().StartsWith(OutputFormats.Json, StringComparison.Ordinal)) + { + using var result = JsonDocument.Parse(_output); + _llm = result.RootElement.GetProperty("llm").Clone(); + } + } + + void Destroy() => Directory.Delete(_folder, recursive: true); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_an_anthropic_environment_endpoint.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_an_anthropic_environment_endpoint.cs new file mode 100644 index 00000000..e0505c3d --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_an_anthropic_environment_endpoint.cs @@ -0,0 +1,40 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_an_anthropic_environment_endpoint : given.an_interpret_command +{ + [Theory] + [InlineData(null, "https://user:secret@environment.example/path?token=hidden", "environment.example")] + [InlineData("", "https://environment.example", "environment.example")] + [InlineData("http://llm:11434", "https://environment.example", "environment.example")] + [InlineData("https://user:secret@configured.example/path?token=hidden", "https://environment.example", "configured.example")] + [InlineData(null, "http://llm:11434", "llm")] + public async Task should_disclose_and_pin_the_effective_endpoint(string? endpoint, string environmentEndpoint, string expectedHost) + { + var previousEndpoint = Environment.GetEnvironmentVariable("ANTHROPIC_BASE_URL"); + try + { + Environment.SetEnvironmentVariable("ANTHROPIC_BASE_URL", environmentEndpoint); + _global = new() { Kind = "anthropic", Endpoint = endpoint, Model = "claude-opus-4-6" }; + + await Interpret(); + + _exitCode.ShouldEqual(ExitCodes.Success); + _noticeBeforeClient.ShouldContain($"endpoint host: {expectedHost}"); + _noticeBeforeRequest.ShouldContain($"endpoint host: {expectedHost}"); + _llm.GetProperty("endpointHost").GetString().ShouldEqual(expectedHost); + _chatClients.Received(1).CreateFor(Arg.Is(options => options.Endpoint == new Uri(string.IsNullOrEmpty(endpoint) || endpoint == "http://llm:11434" ? environmentEndpoint : endpoint).AbsoluteUri)); + _notice.ShouldNotContain("secret"); + _notice.ShouldNotContain("hidden"); + } + finally + { + Environment.SetEnvironmentVariable("ANTHROPIC_BASE_URL", previousEndpoint); + } + } +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_an_invalid_endpoint.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_an_invalid_endpoint.cs new file mode 100644 index 00000000..43fb1e67 --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_an_invalid_endpoint.cs @@ -0,0 +1,31 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; +using Microsoft.Extensions.AI; + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_an_invalid_endpoint : given.an_interpret_command +{ + [Theory] + [InlineData("127.0.0.1:11434")] + [InlineData("")] + [InlineData("relative/path")] + [InlineData("file:///private/captures")] + [InlineData("https://user:secret@")] + public async Task should_reject_before_creating_a_client_or_sending_evidence(string endpoint) + { + Configure(System.Text.Json.JsonSerializer.Serialize(new { llm = new { enabled = true, kind = "Ollama", endpoint } })); + + await Interpret(); + + _exitCode.ShouldEqual(ExitCodes.ValidationError); + _notice.ShouldContain("absolute HTTP or HTTPS URL with a host"); + _notice.ShouldNotContain("secret"); + _chatClients.DidNotReceive().CreateFor(Arg.Any()); + await _client.DidNotReceive().GetResponseAsync(Arg.Any>(), Arg.Any(), Arg.Any()); + File.Exists(_settings.File).ShouldBeFalse(); + } +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_global_configuration.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_global_configuration.cs new file mode 100644 index 00000000..0694c8d7 --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_global_configuration.cs @@ -0,0 +1,17 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_global_configuration : given.an_interpret_command +{ + Task Because() => Interpret(); + + [Fact] void should_succeed() => _exitCode.ShouldEqual(ExitCodes.Success); + [Fact] void should_announce_the_global_provider_before_sending_evidence() => _noticeBeforeRequest.ShouldContain("OpenAI; model: global-model; endpoint host: api.openai.com; source: global config"); + [Fact] void should_report_the_provider_in_json() => _llm.GetProperty("kind").GetString().ShouldEqual("OpenAI"); + [Fact] void should_report_the_global_source_in_json() => _llm.GetProperty("source").GetString().ShouldEqual("global config"); + [Fact] void should_report_refinement_enabled() => _llm.GetProperty("used").GetBoolean().ShouldBeTrue(); + [Fact] void should_not_expose_the_global_secret() => _notice.ShouldNotContain("global-secret"); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_local_disabled.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_local_disabled.cs new file mode 100644 index 00000000..1259fce1 --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_local_disabled.cs @@ -0,0 +1,20 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_local_disabled : given.an_interpret_command +{ + void Establish() => Configure("""{"llm":{"enabled":false}}"""); + Task Because() => Interpret(); + + [Fact] void should_succeed() => _exitCode.ShouldEqual(ExitCodes.Success); + [Fact] void should_write_a_screenplay() => File.ReadAllText(_settings.File).ShouldNotBeEmpty(); + [Fact] void should_not_create_a_chat_client() => _chatClients.DidNotReceive().CreateFor(Arg.Any()); + [Fact] void should_report_no_model_in_json() => _llm.GetProperty("used").GetBoolean().ShouldBeFalse(); + [Fact] void should_report_the_local_source() => _llm.GetProperty("source").GetString().ShouldEqual("local file"); + [Fact] void should_report_heuristics_only() => _notice.ShouldContain("Interpreting with heuristics only."); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_local_enabled.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_local_enabled.cs new file mode 100644 index 00000000..a28b033c --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_local_enabled.cs @@ -0,0 +1,25 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; +using Microsoft.Extensions.AI; + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_local_enabled : given.an_interpret_command +{ + void Establish() => Configure("""{"llm":{"enabled":true,"kind":"OpenAICompatible","modelId":"project-model","endpoint":"https://user:password@project.example/v1?secret=hidden","accessToken":"local-secret"}}"""); + Task Because() => Interpret(); + + [Fact] void should_succeed() => _exitCode.ShouldEqual(ExitCodes.Success); + [Fact] void should_use_the_local_provider() => _chatClients.Received(1).CreateFor(Arg.Is(options => options.ModelId == "project-model")); + [Fact] void should_call_the_model() => _client.Received(1).GetResponseAsync(Arg.Any>(), Arg.Any(), Arg.Any()); + [Fact] void should_announce_before_creating_the_client() => _noticeBeforeClient.ShouldContain("OpenAICompatible; model: project-model; endpoint host: project.example; source: local file"); + [Fact] void should_announce_before_sending_evidence() => _noticeBeforeRequest.ShouldContain("Capture evidence will be sent to this provider."); + [Fact] void should_include_the_model_in_json() => _llm.GetProperty("model").GetString().ShouldEqual("project-model"); + [Fact] void should_include_only_the_endpoint_host_in_json() => _llm.GetProperty("endpointHost").GetString().ShouldEqual("project.example"); + [Fact] void should_not_expose_the_access_token() => _output.ShouldNotContain("local-secret"); + [Fact] void should_not_expose_endpoint_credentials() => _notice.ShouldNotContain("password"); + [Fact] void should_not_expose_endpoint_query_strings() => _notice.ShouldNotContain("hidden"); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_no_llm.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_no_llm.cs new file mode 100644 index 00000000..92596dfb --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_no_llm.cs @@ -0,0 +1,27 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_no_llm : given.an_interpret_command +{ + void Establish() + { + _settings.NoLlm = true; + Configure("""{"llm":{"enabled":true,"kind":"Anthropic","accessToken":"local-secret"}}"""); + } + + Task Because() => Interpret(); + + [Fact] void should_succeed() => _exitCode.ShouldEqual(ExitCodes.Success); + [Fact] void should_write_a_screenplay() => File.ReadAllText(_settings.File).ShouldNotBeEmpty(); + [Fact] void should_not_create_a_chat_client() => _chatClients.DidNotReceive().CreateFor(Arg.Any()); + [Fact] void should_not_load_the_global_configuration() => _globalLoads.ShouldEqual(0); + [Fact] void should_report_no_model_in_json() => _llm.GetProperty("used").GetBoolean().ShouldBeFalse(); + [Fact] void should_report_none_as_the_provider() => _llm.GetProperty("kind").GetString().ShouldEqual("none"); + [Fact] void should_report_the_command_line_source() => _llm.GetProperty("source").GetString().ShouldEqual("--no-llm"); + [Fact] void should_report_heuristics_only() => _notice.ShouldContain("Interpreting with heuristics only."); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_plain_output.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_plain_output.cs new file mode 100644 index 00000000..935fd5ff --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_plain_output.cs @@ -0,0 +1,14 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_plain_output : given.an_interpret_command +{ + void Establish() => _settings.Output = OutputFormats.Plain; + Task Because() => Interpret(); + + [Fact] void should_include_the_provider_in_the_result() => _output.ShouldContain("Language model: OpenAI; model: global-model; endpoint host: api.openai.com; source: global config"); + [Fact] void should_announce_before_sending_evidence() => _noticeBeforeRequest.ShouldContain("source: global config"); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_quiet_output.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_quiet_output.cs new file mode 100644 index 00000000..ee78e05c --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_quiet_output.cs @@ -0,0 +1,18 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_quiet_output : given.an_interpret_command +{ + void Establish() + { + _settings.Quiet = true; + _settings.Output = OutputFormats.Plain; + } + Task Because() => Interpret(); + + [Fact] void should_announce_before_sending_evidence() => _noticeBeforeRequest.ShouldContain("source: global config"); + [Fact] void should_keep_stdout_to_the_output_path() => _output.Trim().ShouldEqual(_settings.File); +} diff --git a/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_table_output_and_no_llm.cs b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_table_output_and_no_llm.cs new file mode 100644 index 00000000..7288c884 --- /dev/null +++ b/Source/Cli.Specs/for_InterpretPrologueCommand/when_interpreting/with_table_output_and_no_llm.cs @@ -0,0 +1,18 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Cratis.Cli.for_InterpretPrologueCommand.when_interpreting; + +[Collection(CliSpecsCollection.Name)] +public class with_table_output_and_no_llm : given.an_interpret_command +{ + void Establish() + { + _settings.Output = OutputFormats.Table; + _settings.NoLlm = true; + } + + Task Because() => Interpret(); + + [Fact] void should_include_none_and_its_source_in_the_result() => _output.ShouldContain("Language model: none; source: --no-llm. Interpreting with heuristics only."); +} diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_anthropic_cli_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_anthropic_cli_configuration.cs index 74ea1eae..8449d041 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_anthropic_cli_configuration.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_anthropic_cli_configuration.cs @@ -9,9 +9,9 @@ public class with_anthropic_cli_configuration : Specification { LlmOptions _result; - void Because() => _result = LlmOptionsResolver.Resolve( + void Because() => _result = LlmOptionsResolver.ResolveWithSource( null, - new LlmConfiguration { Kind = "anthropic", ApiKey = "sk-ant-key", Model = "claude-opus-4-6" }); + new LlmConfiguration { Kind = "anthropic", ApiKey = "sk-ant-key", Model = "claude-opus-4-6" }).Options; [Fact] void should_be_enabled() => _result.Enabled.ShouldBeTrue(); [Fact] void should_map_to_the_anthropic_kind() => _result.Kind.ShouldEqual(LlmKind.Anthropic); diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_disabled_prologue_configuration_and_cli_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_disabled_prologue_configuration_and_cli_configuration.cs index 3ccdf0c3..7d3d1f8f 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_disabled_prologue_configuration_and_cli_configuration.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_disabled_prologue_configuration_and_cli_configuration.cs @@ -9,10 +9,10 @@ public class with_disabled_prologue_configuration_and_cli_configuration : Specif { LlmOptions _result; - void Because() => _result = LlmOptionsResolver.Resolve( - new PrologueConfiguration { Llm = new LlmOptions { Enabled = false } }, - new LlmConfiguration { Kind = "anthropic", ApiKey = "sk-ant-key" }); + void Because() => _result = LlmOptionsResolver.ResolveWithSource( + """{"llm":{"enabled":false}}""", + new LlmConfiguration { Kind = "anthropic", ApiKey = "sk-ant-key" }).Options; - [Fact] void should_fall_back_to_the_cli_configuration() => _result.Kind.ShouldEqual(LlmKind.Anthropic); - [Fact] void should_be_enabled() => _result.Enabled.ShouldBeTrue(); + [Fact] void should_not_fall_back_to_the_cli_configuration() => _result.AccessToken.ShouldNotEqual("sk-ant-key"); + [Fact] void should_be_disabled() => _result.Enabled.ShouldBeFalse(); } diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_enabled_prologue_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_enabled_prologue_configuration.cs index ed7243fc..d13d2a95 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_enabled_prologue_configuration.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_enabled_prologue_configuration.cs @@ -7,17 +7,13 @@ namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; public class with_enabled_prologue_configuration : Specification { - PrologueConfiguration _configuration; LlmOptions _result; - void Establish() => _configuration = new PrologueConfiguration - { - Llm = new LlmOptions { Enabled = true, Kind = LlmKind.Ollama, ModelId = "gemma" } - }; + void Because() => _result = LlmOptionsResolver.ResolveWithSource( + """{"llm":{"enabled":true,"kind":"Ollama","modelId":"gemma"}}""", + new LlmConfiguration { Kind = "anthropic", ApiKey = "sk-ant-key" }).Options; - void Because() => _result = LlmOptionsResolver.Resolve( - _configuration, - new LlmConfiguration { Kind = "anthropic", ApiKey = "sk-ant-key" }); - - [Fact] void should_use_the_prologue_configuration() => _result.ShouldEqual(_configuration.Llm); + [Fact] void should_be_enabled() => _result.Enabled.ShouldBeTrue(); + [Fact] void should_use_the_local_provider() => _result.Kind.ShouldEqual(LlmKind.Ollama); + [Fact] void should_use_the_local_model() => _result.ModelId.ShouldEqual("gemma"); } diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_cli_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_cli_configuration.cs index 1db7d1ea..0fb937a4 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_cli_configuration.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_cli_configuration.cs @@ -9,9 +9,9 @@ public class with_local_cli_configuration : Specification { LlmOptions _result; - void Because() => _result = LlmOptionsResolver.Resolve( + void Because() => _result = LlmOptionsResolver.ResolveWithSource( null, - new LlmConfiguration { Kind = "local", Endpoint = "http://localhost:11434/v1", Model = "llama3" }); + new LlmConfiguration { Kind = "local", Endpoint = "http://localhost:11434/v1", Model = "llama3" }).Options; [Fact] void should_be_enabled() => _result.Enabled.ShouldBeTrue(); [Fact] void should_map_to_the_openai_compatible_kind() => _result.Kind.ShouldEqual(LlmKind.OpenAICompatible); diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_disabled.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_disabled.cs new file mode 100644 index 00000000..ecfa6bda --- /dev/null +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_disabled.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; + +public class with_local_disabled : Specification +{ + (LlmOptions Options, string Source) _result; + + void Because() => _result = LlmOptionsResolver.ResolveWithSource("""{"Llm":{"Enabled":false}}""", new() { Kind = "openai", Model = "global-model" }); + + [Fact] void should_disable_refinement() => _result.Options.Enabled.ShouldBeFalse(); + [Fact] void should_name_the_local_source() => _result.Source.ShouldEqual("local file"); +} diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_enabled.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_enabled.cs new file mode 100644 index 00000000..52d567a9 --- /dev/null +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_enabled.cs @@ -0,0 +1,18 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; + +public class with_local_enabled : Specification +{ + (LlmOptions Options, string Source) _result; + + void Because() => _result = LlmOptionsResolver.ResolveWithSource("""{"llm":{"enabled":true,"kind":"Ollama","modelId":"project-model","endpoint":"http://localhost:11434"}}""", new() { Kind = "openai", Model = "global-model" }); + + [Fact] void should_enable_refinement() => _result.Options.Enabled.ShouldBeTrue(); + [Fact] void should_use_the_local_provider() => _result.Options.Kind.ShouldEqual(LlmKind.Ollama); + [Fact] void should_use_the_local_model() => _result.Options.ModelId.ShouldEqual("project-model"); + [Fact] void should_name_the_local_source() => _result.Source.ShouldEqual("local file"); +} diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_enabled_unset.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_enabled_unset.cs new file mode 100644 index 00000000..1903f68e --- /dev/null +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_local_enabled_unset.cs @@ -0,0 +1,17 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; + +public class with_local_enabled_unset : Specification +{ + (LlmOptions Options, string Source) _result; + + void Because() => _result = LlmOptionsResolver.ResolveWithSource("""{"llm":{"kind":"Ollama","modelId":"project-model"}}""", new() { Kind = "openai", Model = "global-model" }); + + [Fact] void should_enable_refinement() => _result.Options.Enabled.ShouldBeTrue(); + [Fact] void should_use_the_global_model() => _result.Options.ModelId.ShouldEqual("global-model"); + [Fact] void should_name_the_global_source() => _result.Source.ShouldEqual("global config"); +} diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_no_llm.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_no_llm.cs new file mode 100644 index 00000000..92e99b1a --- /dev/null +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_no_llm.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; + +public class with_no_llm : Specification +{ + (LlmOptions Options, string Source) _result; + + void Because() => _result = LlmOptionsResolver.ResolveWithSource("""{"llm":{"enabled":true}}""", new() { Kind = "openai" }, noLlm: true); + + [Fact] void should_disable_refinement() => _result.Options.Enabled.ShouldBeFalse(); + [Fact] void should_name_the_command_line_source() => _result.Source.ShouldEqual("--no-llm"); +} diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_openai_cli_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_openai_cli_configuration.cs index a43862b4..dc6427aa 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_openai_cli_configuration.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_openai_cli_configuration.cs @@ -9,9 +9,9 @@ public class with_openai_cli_configuration : Specification { LlmOptions _result; - void Because() => _result = LlmOptionsResolver.Resolve( + void Because() => _result = LlmOptionsResolver.ResolveWithSource( null, - new LlmConfiguration { Kind = "openai", ApiKey = "sk-key" }); + new LlmConfiguration { Kind = "openai", ApiKey = "sk-key" }).Options; [Fact] void should_be_enabled() => _result.Enabled.ShouldBeTrue(); [Fact] void should_map_to_the_openai_kind() => _result.Kind.ShouldEqual(LlmKind.OpenAI); diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_unknown_cli_kind.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_unknown_cli_kind.cs index 51be6c25..8f87000e 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_unknown_cli_kind.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/with_unknown_cli_kind.cs @@ -9,7 +9,7 @@ public class with_unknown_cli_kind : Specification { LlmOptions _result; - void Because() => _result = LlmOptionsResolver.Resolve(null, new LlmConfiguration { Kind = "gemini" }); + void Because() => _result = LlmOptionsResolver.ResolveWithSource(null, new LlmConfiguration { Kind = "gemini" }).Options; [Fact] void should_not_be_enabled() => _result.Enabled.ShouldBeFalse(); } diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_a_local_section.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_a_local_section.cs new file mode 100644 index 00000000..dd1e3dd0 --- /dev/null +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_a_local_section.cs @@ -0,0 +1,17 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; + +public class without_a_local_section : Specification +{ + (LlmOptions Options, string Source) _result; + + void Because() => _result = LlmOptionsResolver.ResolveWithSource("{}", new() { Kind = "anthropic", Model = "global-model" }); + + [Fact] void should_use_the_global_provider() => _result.Options.Kind.ShouldEqual(LlmKind.Anthropic); + [Fact] void should_enable_refinement() => _result.Options.Enabled.ShouldBeTrue(); + [Fact] void should_name_the_global_source() => _result.Source.ShouldEqual("global config"); +} diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_any_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_any_configuration.cs index b0792811..2ab4ccf4 100644 --- a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_any_configuration.cs +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_any_configuration.cs @@ -9,7 +9,7 @@ public class without_any_configuration : Specification { LlmOptions _result; - void Because() => _result = LlmOptionsResolver.Resolve(null, null); + void Because() => _result = LlmOptionsResolver.ResolveWithSource(null, null).Options; [Fact] void should_not_be_enabled() => _result.Enabled.ShouldBeFalse(); } diff --git a/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_configuration.cs b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_configuration.cs new file mode 100644 index 00000000..03ea3ea6 --- /dev/null +++ b/Source/Cli.Specs/for_LlmOptionsResolver/when_resolving/without_configuration.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmOptionsResolver.when_resolving; + +public class without_configuration : Specification +{ + (LlmOptions Options, string Source) _result; + + void Because() => _result = LlmOptionsResolver.ResolveWithSource(null, null); + + [Fact] void should_disable_refinement() => _result.Options.Enabled.ShouldBeFalse(); + [Fact] void should_name_no_source() => _result.Source.ShouldEqual("none"); +} diff --git a/Source/Cli.Specs/for_LlmUsage/when_describing/with_a_custom_anthropic_endpoint.cs b/Source/Cli.Specs/for_LlmUsage/when_describing/with_a_custom_anthropic_endpoint.cs new file mode 100644 index 00000000..fcfe2456 --- /dev/null +++ b/Source/Cli.Specs/for_LlmUsage/when_describing/with_a_custom_anthropic_endpoint.cs @@ -0,0 +1,18 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmUsage.when_describing; + +public class with_a_custom_anthropic_endpoint : Specification +{ + LlmUsage _result; + + void Because() => _result = LlmUsage.From(new LlmOptions { Enabled = true, Kind = LlmKind.Anthropic, Endpoint = "https://user:secret@gateway.example/path?token=hidden" }, "local file"); + + [Fact] void should_name_only_the_custom_host() => _result.EndpointHost.ShouldEqual("gateway.example"); + [Fact] void should_not_expose_credentials_in_the_notice() => _result.Notice.ShouldNotContain("secret"); + [Fact] void should_not_expose_paths_in_the_notice() => _result.Notice.ShouldNotContain("/path"); + [Fact] void should_not_expose_query_strings_in_the_notice() => _result.Notice.ShouldNotContain("hidden"); +} diff --git a/Source/Cli.Specs/for_LlmUsage/when_describing/with_an_openai_environment_endpoint.cs b/Source/Cli.Specs/for_LlmUsage/when_describing/with_an_openai_environment_endpoint.cs new file mode 100644 index 00000000..1f1e2fe0 --- /dev/null +++ b/Source/Cli.Specs/for_LlmUsage/when_describing/with_an_openai_environment_endpoint.cs @@ -0,0 +1,25 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; + +namespace Cratis.Cli.for_LlmUsage.when_describing; + +[Collection(CliSpecsCollection.Name)] +public class with_an_openai_environment_endpoint : Specification +{ + string? _previousEndpoint; + LlmUsage _result; + + void Establish() + { + _previousEndpoint = Environment.GetEnvironmentVariable("OPENAI_BASE_URL"); + Environment.SetEnvironmentVariable("OPENAI_BASE_URL", "https://environment.example"); + } + + void Because() => _result = LlmUsage.From(new LlmOptions { Enabled = true, Kind = LlmKind.OpenAI }, "global config"); + + [Fact] void should_match_the_clients_public_endpoint() => _result.EndpointHost.ShouldEqual("api.openai.com"); + + void Destroy() => Environment.SetEnvironmentVariable("OPENAI_BASE_URL", _previousEndpoint); +} diff --git a/Source/Cli.Specs/for_LlmUsage/when_describing/with_anthropic_defaults.cs b/Source/Cli.Specs/for_LlmUsage/when_describing/with_anthropic_defaults.cs new file mode 100644 index 00000000..aa3eab65 --- /dev/null +++ b/Source/Cli.Specs/for_LlmUsage/when_describing/with_anthropic_defaults.cs @@ -0,0 +1,27 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; +using Cratis.Prologue.Interpretation; + +namespace Cratis.Cli.for_LlmUsage.when_describing; + +[Collection(CliSpecsCollection.Name)] +public class with_anthropic_defaults : Specification +{ + string? _previousEndpoint; + LlmUsage _result; + + void Establish() + { + _previousEndpoint = Environment.GetEnvironmentVariable("ANTHROPIC_BASE_URL"); + Environment.SetEnvironmentVariable("ANTHROPIC_BASE_URL", null); + } + + void Because() => _result = LlmUsage.From(new LlmOptions { Enabled = true, Kind = LlmKind.Anthropic, ModelId = string.Empty }, "global config"); + + [Fact] void should_name_the_default_model() => _result.Model.ShouldEqual(LlmChatClient.DefaultAnthropicModelId); + [Fact] void should_name_the_actual_public_endpoint() => _result.EndpointHost.ShouldEqual("api.anthropic.com"); + + void Destroy() => Environment.SetEnvironmentVariable("ANTHROPIC_BASE_URL", _previousEndpoint); +} diff --git a/Source/Cli.Specs/for_LlmUsage/when_describing/with_openai_defaults.cs b/Source/Cli.Specs/for_LlmUsage/when_describing/with_openai_defaults.cs new file mode 100644 index 00000000..63f8bb4f --- /dev/null +++ b/Source/Cli.Specs/for_LlmUsage/when_describing/with_openai_defaults.cs @@ -0,0 +1,17 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Prologue.Configuration; +using Cratis.Prologue.Interpretation; + +namespace Cratis.Cli.for_LlmUsage.when_describing; + +public class with_openai_defaults : Specification +{ + LlmUsage _result; + + void Because() => _result = LlmUsage.From(new LlmOptions { Enabled = true, Kind = LlmKind.OpenAI, ModelId = string.Empty, Endpoint = "https://ignored.example/secret" }, "global config"); + + [Fact] void should_name_the_default_model() => _result.Model.ShouldEqual(LlmChatClient.DefaultOpenAIModelId); + [Fact] void should_name_the_actual_public_endpoint() => _result.EndpointHost.ShouldEqual("api.openai.com"); +} diff --git a/Source/Cli/Commands/Prologue/InterpretPrologueCommand.cs b/Source/Cli/Commands/Prologue/InterpretPrologueCommand.cs index 2de9ab41..3f20fbb1 100644 --- a/Source/Cli/Commands/Prologue/InterpretPrologueCommand.cs +++ b/Source/Cli/Commands/Prologue/InterpretPrologueCommand.cs @@ -16,7 +16,7 @@ namespace Cratis.Cli.Commands.Prologue; /// interpreter session (optionally refined by the configured language model, asking questions along the way), /// and writes the resulting .play file. /// -[LlmDescription("Interprets Prologue capture (.jsonl) files into a Cratis Screenplay (.play) file. Uses deterministic heuristics, refined by the configured language model when one is set up (cratis-prologue.json Llm section or cratis llm use). In an interactive terminal the language model may ask clarifying questions; non-interactive runs never ask. Writes .play to the current directory unless --file is given.")] +[LlmDescription("Interprets Prologue capture (.jsonl) files into a Cratis Screenplay (.play) file. Uses deterministic heuristics, optionally refined by a language model. --no-llm or local llm.enabled: false prevents all model requests; an unset local enabled setting falls back to cratis llm use. Announces the provider, model, endpoint host and source to stderr before sending evidence. In an interactive terminal the language model may ask clarifying questions; non-interactive runs never ask. Writes .play to the current directory unless --file is given.")] [CommandEffect(CommandEffect.Local)] [CliCommand("interpret", "Interpret captured system behavior into a Screenplay", Branch = typeof(PrologueBranch))] [CliExample("prologue", "interpret")] @@ -25,9 +25,26 @@ namespace Cratis.Cli.Commands.Prologue; [LlmOption("[PATH]", "string", "Folder holding the capture (.jsonl) files. Defaults to the configured JSON output directory when a cratis-prologue.json is found, otherwise the current directory.")] [LlmOption("--file", "string", "File to write the generated Screenplay to. Defaults to .play in the current directory.")] [LlmOption("--prologue-id", "guid", "The Prologue the captures belong to. Defaults from cratis-prologue.json when present.")] -[LlmOutputAdvice("json", "JSON outputs the written path, system name, and module/feature/slice counts as one object.")] +[LlmOption("--no-llm", "bool", "Force heuristics-only interpretation. Never creates a chat client or sends capture evidence to a model, regardless of local or global configuration.")] +[LlmOutputAdvice("json", "JSON outputs the written path, system name, module/feature/slice counts, and llm block (used, kind, model, endpointHost, source) as one object. The provider notice is written to stderr before interpretation.")] public class InterpretPrologueCommand : AsyncCommand { + readonly IChatClientFactory _chatClients; + readonly Func _loadLlmConfiguration; + + /// + /// Initializes a new instance of the class. + /// + public InterpretPrologueCommand() : this(new ChatClientFactory(), () => CliConfiguration.Load().Llm) + { + } + + internal InterpretPrologueCommand(IChatClientFactory chatClients, Func loadLlmConfiguration) + { + _chatClients = chatClients; + _loadLlmConfiguration = loadLlmConfiguration; + } + /// public override async Task ExecuteAsync(CommandContext context, InterpretPrologueSettings settings, CancellationToken cancellationToken) { @@ -35,7 +52,8 @@ public override async Task ExecuteAsync(CommandContext context, InterpretPr var currentDirectory = Directory.GetCurrentDirectory(); var configurationPath = PrologueConfigurationFiles.Find(settings.Path, currentDirectory); - var configuration = configurationPath is not null ? await PrologueConfigurationFile.ReadFromFile(configurationPath) : null; + var configurationJson = configurationPath is not null ? await File.ReadAllTextAsync(configurationPath, cancellationToken) : null; + var configuration = configurationJson is not null ? PrologueConfigurationFile.Read(configurationJson) : null; var folder = CaptureFolders.Resolve(settings.Path, configuration, configurationPath, currentDirectory); var prologueId = settings.PrologueId ?? configuration?.Prologue.PrologueId ?? Guid.Empty; @@ -50,16 +68,27 @@ public override async Task ExecuteAsync(CommandContext context, InterpretPr return ExitCodes.NotFound; } - var llmOptions = LlmOptionsResolver.Resolve(configuration, CliConfiguration.Load().Llm); - var showStatus = string.Equals(format, OutputFormats.Table, StringComparison.Ordinal) || - string.Equals(format, OutputFormats.Plain, StringComparison.Ordinal); - if (!llmOptions.Enabled && showStatus) + var (llmOptions, llmSource) = settings.NoLlm + ? LlmOptionsResolver.ResolveWithSource(null, null, noLlm: true) + : LlmOptionsResolver.ResolveWithSource(configurationJson, _loadLlmConfiguration()); + LlmUsage llm; + try + { + llm = LlmUsage.From(llmOptions, llmSource); + } + catch (InvalidLlmEndpoint error) { - AnsiConsole.MarkupLine($" [{OutputFormatter.Muted.ToMarkup()}]No language model configured — interpreting with heuristics only. Configure one with: cratis llm use [/]"); + OutputFormatter.WriteError(format, error.Message, "Configure an endpoint such as http://127.0.0.1:11434, or use --no-llm for heuristics only.", ExitCodes.ValidationErrorCode); + return ExitCodes.ValidationError; } + // Always announce before creating a session, including JSON, quiet and unattended runs. + await Console.Error.WriteLineAsync(llm.Notice); + var showStatus = string.Equals(format, OutputFormats.Table, StringComparison.Ordinal) || + string.Equals(format, OutputFormats.Plain, StringComparison.Ordinal); + var callbacks = new ConsoleInterpreterCallbacks(llmOptions, showStatus, cancellationToken); - var factory = new InterpreterSessionFactory(new HeuristicModelBuilder(), new ChatClientFactory(), NullLogger.Instance); + var factory = new InterpreterSessionFactory(new HeuristicModelBuilder(), _chatClients, NullLogger.Instance); var interactive = AnsiConsole.Profile.Capabilities.Interactive && !settings.Yes; var session = factory.CreateNew( prologueId, @@ -83,11 +112,11 @@ public override async Task ExecuteAsync(CommandContext context, InterpretPr var outputPath = ScreenplayOutput.ResolvePath(settings.File, state.Model.SystemName, currentDirectory); await File.WriteAllTextAsync(outputPath, source, cancellationToken); - WriteResult(format, state.Model, outputPath); + WriteResult(format, state.Model, outputPath, llm); return ExitCodes.Success; } - static void WriteResult(string format, ExtractionResult model, string outputPath) + static void WriteResult(string format, ExtractionResult model, string outputPath, LlmUsage llm) { var features = model.Modules.Sum(module => module.Features.Sum(CountFeatures)); var slices = model.Modules.Sum(module => module.Features.Sum(CountSlices)); @@ -106,7 +135,8 @@ static void WriteResult(string format, ExtractionResult model, string outputPath model.SystemName, Modules = model.Modules.Count, Features = features, - Slices = slices + Slices = slices, + Llm = llm }, result => { @@ -115,7 +145,8 @@ static void WriteResult(string format, ExtractionResult model, string outputPath $"System: {result.SystemName.EscapeMarkup()}\n" + $"Modules: {result.Modules}\n" + $"Features: {result.Features}\n" + - $"Slices: {result.Slices}"); + $"Slices: {result.Slices}\n" + + result.Llm.Notice.EscapeMarkup()); var panel = new Panel(content) .Header(" Screenplay generated ") .Border(BoxBorder.Rounded) diff --git a/Source/Cli/Commands/Prologue/InterpretPrologueSettings.cs b/Source/Cli/Commands/Prologue/InterpretPrologueSettings.cs index a1465f21..84333662 100644 --- a/Source/Cli/Commands/Prologue/InterpretPrologueSettings.cs +++ b/Source/Cli/Commands/Prologue/InterpretPrologueSettings.cs @@ -28,4 +28,11 @@ public class InterpretPrologueSettings : GlobalSettings [CommandOption("--prologue-id ")] [Description("The Prologue the captures belong to. Defaults from cratis-prologue.json when one is present in PATH or the current directory.")] public Guid? PrologueId { get; set; } + + /// + /// Gets or sets a value indicating whether interpretation must use heuristics only. + /// + [CommandOption("--no-llm")] + [Description("Force heuristics-only interpretation without creating a chat client or sending capture evidence to a model, regardless of local or global configuration.")] + public bool NoLlm { get; set; } } diff --git a/Source/Cli/Commands/Prologue/InvalidLlmEndpoint.cs b/Source/Cli/Commands/Prologue/InvalidLlmEndpoint.cs new file mode 100644 index 00000000..585f184e --- /dev/null +++ b/Source/Cli/Commands/Prologue/InvalidLlmEndpoint.cs @@ -0,0 +1,9 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Cratis.Cli.Commands.Prologue; + +/// +/// The exception that is thrown when a language-model destination cannot be safely identified. +/// +public class InvalidLlmEndpoint() : Exception("The language-model endpoint must be an absolute HTTP or HTTPS URL with a host. No capture evidence was sent."); diff --git a/Source/Cli/Commands/Prologue/LlmOptionsResolver.cs b/Source/Cli/Commands/Prologue/LlmOptionsResolver.cs index 549d9546..cc3d344a 100644 --- a/Source/Cli/Commands/Prologue/LlmOptionsResolver.cs +++ b/Source/Cli/Commands/Prologue/LlmOptionsResolver.cs @@ -1,37 +1,48 @@ // Copyright (c) Cratis. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Diagnostics.CodeAnalysis; using Cratis.Cli.Commands.Llm; using Cratis.Prologue.Configuration; namespace Cratis.Cli.Commands.Prologue; /// -/// Resolves the the interpreter refines with — the Llm section of a -/// cratis-prologue.json when it is enabled there, otherwise the llm section of the CLI's own -/// configuration (written by cratis llm use), and disabled refinement when neither is configured. +/// Resolves language-model refinement with command-line disablement first, explicit local settings next, +/// and the CLI's global configuration only when the local enabled setting is absent. /// public static class LlmOptionsResolver { /// - /// Resolves the language-model options for an interpretation run. + /// Resolves refinement and its source while preserving the distinction between absent and false local settings. /// - /// The cratis-prologue.json configuration found for the run; when none exists. + /// The local configuration JSON; when none exists. /// The CLI's language model configuration; when not configured. - /// The resolved ; disabled when nothing is configured. - public static LlmOptions Resolve(PrologueConfiguration? prologueConfiguration, LlmConfiguration? cliConfiguration) + /// Whether the command line forces heuristics-only interpretation. + /// The resolved options and their source; disabled when requested or nothing is configured. + public static (LlmOptions Options, string Source) ResolveWithSource([StringSyntax(StringSyntaxAttribute.Json)] string? prologueConfigurationJson, LlmConfiguration? cliConfiguration, bool noLlm = false) { - if (prologueConfiguration?.Llm.Enabled == true) + if (noLlm) { - return prologueConfiguration.Llm; + return (new LlmOptions { Enabled = false }, "--no-llm"); + } + + // The package's non-nullable Enabled property cannot distinguish an absent setting from false. + // Use the same serializer options so property casing and duplicate-property precedence agree. + var localEnabled = prologueConfigurationJson is not null + ? JsonSerializer.Deserialize(prologueConfigurationJson, PrologueConfigurationFile.SerializerOptions)?.Llm?.Enabled + : null; + if (localEnabled is { } enabled) + { + return (enabled ? PrologueConfigurationFile.Read(prologueConfigurationJson!).Llm : new LlmOptions { Enabled = false }, "local file"); } if (KindFor(cliConfiguration) is { } kind) { - return FromCli(cliConfiguration!, kind); + return (FromCli(cliConfiguration!, kind), "global config"); } - return new LlmOptions { Enabled = false }; + return (new LlmOptions { Enabled = false }, "none"); } static LlmKind? KindFor(LlmConfiguration? configuration) => @@ -65,4 +76,8 @@ static LlmOptions FromCli(LlmConfiguration configuration, LlmKind kind) return options; } + + sealed record LocalConfiguration(LocalLlm? Llm); + + sealed record LocalLlm(bool? Enabled); } diff --git a/Source/Cli/Commands/Prologue/LlmUsage.cs b/Source/Cli/Commands/Prologue/LlmUsage.cs new file mode 100644 index 00000000..6003283e --- /dev/null +++ b/Source/Cli/Commands/Prologue/LlmUsage.cs @@ -0,0 +1,65 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json.Serialization; +using Anthropic.Core; +using Cratis.Prologue.Configuration; +using Cratis.Prologue.Interpretation; + +namespace Cratis.Cli.Commands.Prologue; + +/// +/// Describes the effective language model without exposing credentials or endpoint paths. +/// +/// Whether language-model refinement is enabled for this run. +/// The provider kind, or none. +/// The effective model identifier, or an empty string. +/// The effective endpoint host, or an empty string. +/// Where the refinement setting came from. +public record LlmUsage(bool Used, string Kind, string Model, string EndpointHost, string Source) +{ + /// + /// Gets the notice shown before any capture evidence can be sent. + /// + [JsonIgnore] + public string Notice => Used + ? $"Language model: {Kind}; model: {Model}; endpoint host: {EndpointHost}; source: {Source}. Capture evidence will be sent to this provider." + : $"Language model: none; source: {Source}. Interpreting with heuristics only."; + + /// + /// Resolves and validates the endpoint, pins it in the client options, and describes the selected provider. + /// + /// The resolved language-model options. + /// The source of the options. + /// The credential-free provider description. + /// The endpoint is not an absolute HTTP or HTTPS URL with a host. + public static LlmUsage From(LlmOptions options, string source) + { + if (!options.Enabled) + { + return new(false, "none", string.Empty, string.Empty, source); + } + + // The pinned SDK's OpenAI client has no environment-based endpoint override. Other providers + // receive an explicit endpoint. Anthropic alone reads ANTHROPIC_BASE_URL when its endpoint is unset. + var endpoint = options.Kind switch + { + LlmKind.OpenAI => "https://api.openai.com/v1", + LlmKind.Anthropic when string.IsNullOrEmpty(options.Endpoint) || string.Equals(options.Endpoint, "http://llm:11434", StringComparison.OrdinalIgnoreCase) => + Environment.GetEnvironmentVariable("ANTHROPIC_BASE_URL") ?? EnvironmentUrl.Production, + _ => options.Endpoint + }; + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var destination) || + (destination.Scheme != Uri.UriSchemeHttp && destination.Scheme != Uri.UriSchemeHttps) || + string.IsNullOrEmpty(destination.Host)) + { + throw new InvalidLlmEndpoint(); + } + + // AbsoluteUri also adds a trailing slash to the default Ollama URL if an Anthropic environment + // override chooses it, so CreateAnthropic treats it as explicit rather than rereading the environment. + options.Endpoint = destination.AbsoluteUri; + + return new(true, options.Kind.ToString(), LlmChatClient.EffectiveModelId(options), destination.Host, source); + } +}