From b5aa4da50b35ef6f71fff1861ed66ae32899b783 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Thu, 8 Oct 2026 08:39:31 +0300 Subject: [PATCH 1/2] Correct what the scenarios say about ClojureCLR, Basilisp and jank no-stdin said reading input gets an empty string right away, like on Basilisp. That's only half the story. Basilisp and ClojureCLR have no stdin op and read their own stdin instead, so reading input waits for the server's terminal, or gets the end of input right away when the server was started in the background (an empty string on Basilisp, nil on ClojureCLR). no-stdin now gets nil, which is what Clojure gives at the end of input, and names ClojureCLR as well. jank doesn't have a stdin op either, but its read-line throws ("TODO: port read-line"), which a client sees as an eval error rather than a value. That's the new read-line-throws scenario. ClojureCLR also has no interrupt op, and the version it puts in versions is a plain string, like Babashka's. --- doc/usage.md | 7 ++++--- internal/serve/matrix_test.go | 3 ++- internal/serve/scenarios.go | 29 +++++++++++++++++++++++------ internal/serve/serve.go | 12 ++++++++---- internal/serve/serve_test.go | 12 ++++++++---- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/doc/usage.md b/doc/usage.md index 18d8f44..3eb1dfc 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -320,9 +320,10 @@ the scenarios, and every one of them is something a real server does | `error-with-done` | `eval-error` comes in the same message as `done` | jank | | `no-op-echo` | Replies to unknown ops don't say which op it was | ClojureCLR, Basilisp, jank, dialtone, repartee | | `no-close-op` | `describe` doesn't list `close`, even though `close` works | jank | -| `no-interrupt` | There's no `interrupt` op | Basilisp, jank | -| `no-stdin` | There's no `stdin` op, and reading input gets an empty string right away | Basilisp | -| `string-versions` | `versions.proof` is a plain string rather than a dict | Babashka, for `versions.babashka` | +| `no-interrupt` | There's no `interrupt` op | ClojureCLR, Basilisp, jank | +| `no-stdin` | There's no `stdin` op, and reading input gets `nil` right away | ClojureCLR and Basilisp started in the background, as they read their own stdin, where Basilisp gets an empty string | +| `read-line-throws` | There's no `stdin` op, and reading input throws | jank | +| `string-versions` | `versions.proof` is a plain string rather than a dict | Babashka for `versions.babashka`, ClojureCLR for `versions.clojure.tools.nrepl` | | `no-session-closed` | `close` replies with `done` alone | Basilisp, jank, dialtone, repartee | | `shared-state` | Sessions on the same connection share `*1`, `*e` and the current namespace | ClojureCLR, Basilisp, jank | | `socket-sessions` | A session only exists on the connection that cloned it | ClojureCLR, Basilisp, jank | diff --git a/internal/serve/matrix_test.go b/internal/serve/matrix_test.go index d3685a6..ac6b0ad 100644 --- a/internal/serve/matrix_test.go +++ b/internal/serve/matrix_test.go @@ -59,6 +59,7 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) { {[]string{"no-close-op"}, map[string]check.Verdict{"describe.required-ops": F}}, {[]string{"no-interrupt"}, nil}, {[]string{"no-stdin"}, noStdin}, + {[]string{"read-line-throws"}, noStdin}, {[]string{"string-versions"}, nil}, {[]string{"no-session-closed"}, map[string]check.Verdict{"session.close": F}}, {[]string{"shared-state"}, map[string]check.Verdict{"session.isolated": F}}, @@ -78,7 +79,7 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) { "session.close": F, "session.unknown": F, "session.closed": F, "eval.stderr": F, "eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S}}, {[]string{"no-close-op", "no-op-echo", "socket-sessions", "shared-state", "no-session-closed", "any-session", - "last-value", "ns-fallback", "no-stdin", "no-interrupt", "unsorted-keys", "empty-messages", "error-with-done"}, // jank + "last-value", "ns-fallback", "read-line-throws", "no-interrupt", "unsorted-keys", "empty-messages", "error-with-done"}, // jank map[string]check.Verdict{"describe.required-ops": F, "op.unknown-echo": W, "session.across-connections": W, "session.isolated": F, "session.close": F, "session.unknown": F, "session.closed": F, "eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, diff --git a/internal/serve/scenarios.go b/internal/serve/scenarios.go index 10dbde9..23e1924 100644 --- a/internal/serve/scenarios.go +++ b/internal/serve/scenarios.go @@ -20,12 +20,25 @@ type Scenario struct { // behavior is the set of scenarios a server runs with. type behavior struct { splitOutput, emptyMessages, lastValue, noErr, errorWithDone bool - noOpEcho, noCloseOp, noInterrupt, noStdin, stringVersions bool + noOpEcho, noCloseOp, noInterrupt, stringVersions bool noSessionClosed, sharedState, socketSessions, anySession bool nsFallback, nsError, eofError bool unsortedKeys, byteWrites, batchedWrites, hangUp bool + stdin stdinMode } +// stdinMode is what reading input does. +type stdinMode int + +const ( + // askForInput is what nREPL does: ask with need-input, then read what + // the stdin op sends. + askForInput stdinMode = iota + // The other two are for servers without a stdin op. + endOfInput + throwOnRead +) + // scenario is a Scenario and the behavior it sets. type scenario struct { Scenario @@ -48,11 +61,15 @@ var catalog = []scenario{ func(b *behavior) { b.noOpEcho = true }}, {Scenario{"no-close-op", "describe doesn't list close, even though close works", "jank"}, func(b *behavior) { b.noCloseOp = true }}, - {Scenario{"no-interrupt", "There's no interrupt op", "Basilisp, jank"}, + {Scenario{"no-interrupt", "There's no interrupt op", "ClojureCLR, Basilisp, jank"}, func(b *behavior) { b.noInterrupt = true }}, - {Scenario{"no-stdin", "There's no stdin op, and reading input gets an empty string right away", "Basilisp"}, - func(b *behavior) { b.noStdin = true }}, - {Scenario{"string-versions", "versions.proof is a plain string rather than a dict", "Babashka, for versions.babashka"}, + {Scenario{"no-stdin", "There's no stdin op, and reading input gets nil right away", + "ClojureCLR and Basilisp started in the background, as they read their own stdin, where Basilisp gets an empty string"}, + func(b *behavior) { b.stdin = endOfInput }}, + {Scenario{"read-line-throws", "There's no stdin op, and reading input throws", "jank"}, + func(b *behavior) { b.stdin = throwOnRead }}, + {Scenario{"string-versions", "versions.proof is a plain string rather than a dict", + "Babashka for versions.babashka, ClojureCLR for versions.clojure.tools.nrepl"}, func(b *behavior) { b.stringVersions = true }}, {Scenario{"no-session-closed", "close replies with done alone, without session-closed", "Basilisp, jank, dialtone, repartee"}, func(b *behavior) { b.noSessionClosed = true }}, @@ -88,7 +105,7 @@ func Scenarios() []Scenario { } // conflicts are scenarios that can't be combined. -var conflicts = [][2]string{{"ns-fallback", "ns-error"}, {"byte-writes", "batched-writes"}} +var conflicts = [][2]string{{"ns-fallback", "ns-error"}, {"byte-writes", "batched-writes"}, {"no-stdin", "read-line-throws"}} func behave(names []string) (behavior, error) { var b behavior diff --git a/internal/serve/serve.go b/internal/serve/serve.go index 2cfaf4b..35ca836 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -220,7 +220,7 @@ func (s *Server) offers(op string) bool { case "interrupt": return !s.b.noInterrupt case "stdin": - return !s.b.noStdin + return s.b.stdin == askForInput } return slices.Contains(ops, op) } @@ -467,9 +467,13 @@ func (s *Server) runEval(sess *session, j *job) { } }, readLine: func(interrupts <-chan struct{}) (any, error) { - if s.b.noStdin { - // Basilisp's read-line gets an empty string right away. - return "", nil + switch s.b.stdin { + case throwOnRead: + // What jank says. + return nil, throwf("clojure.lang.ExceptionInfo", "TODO: port read-line") + case endOfInput: + // The end of the server's own stdin. + return nil, nil } line, ok, err := sess.readLine(s.ctx, interrupts, func() { reply(map[string]any{"status": []any{"need-input"}}) }) switch { diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go index 4e4a99e..df050ff 100644 --- a/internal/serve/serve_test.go +++ b/internal/serve/serve_test.go @@ -189,7 +189,9 @@ func TestInput(t *testing.T) { {scenario: "eof-error", inputs: []string{"", ""}, err: strings.Repeat("Execution error (ClassCastException) at user/eval1 (REPL:1).\n"+ "class java.lang.Long cannot be cast to class java.lang.Character\n", 2)}, - {scenario: "no-stdin", values: []string{`""`, `""`}}, + {scenario: "no-stdin", values: []string{"nil", "nil"}}, + {scenario: "read-line-throws", + err: strings.Repeat("Execution error (ExceptionInfo) at user/eval1 (REPL:1).\nTODO: port read-line\n", 2)}, } for _, tc := range cases { t.Run(strings.Join(append([]string{tc.scenario}, tc.inputs...), "|"), func(t *testing.T) { @@ -380,6 +382,8 @@ func TestDescribe(t *testing.T) { {nil, []string{"clone", "close", "describe", "eval", "interrupt", "stdin"}, map[string]any{"major": int64(0), "minor": int64(1), "incremental": int64(0), "version-string": "0.1.0-dev"}}, {[]string{"no-close-op", "no-interrupt", "no-stdin", "string-versions"}, []string{"clone", "describe", "eval"}, "0.1.0-dev"}, + {[]string{"read-line-throws"}, []string{"clone", "close", "describe", "eval", "interrupt"}, + map[string]any{"major": int64(0), "minor": int64(1), "incremental": int64(0), "version-string": "0.1.0-dev"}}, } for _, tc := range cases { c := connect(t, serveFor(t, tc.scenarios...)) @@ -396,10 +400,10 @@ func TestDescribe(t *testing.T) { } func TestOpsTheScenariosTakeAway(t *testing.T) { - c := connect(t, serveFor(t, "no-interrupt", "no-stdin")) - for _, op := range []string{"interrupt", "stdin"} { + for scenario, op := range map[string]string{"no-interrupt": "interrupt", "no-stdin": "stdin", "read-line-throws": "stdin"} { + c := connect(t, serveFor(t, scenario)) if r := request(t, c, nrepl.Message{"op": op, "session": clone(t, c)}); !r.HasStatus("unknown-op") { - t.Errorf("%s: %v", op, r.Status()) + t.Errorf("%s: %s got %v", scenario, op, r.Status()) } } } From a2cdca061282e91ec3ad142997950d0f592fb6c7 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Thu, 8 Oct 2026 08:39:55 +0300 Subject: [PATCH 2/2] Let proof serve behave like a whole server Testing a client against something like jank meant looking up which scenarios jank does and listing them all. Now -like jank turns them on at once, and the same goes for every other server proof has a profile for. More scenarios can follow as before (e.g. -like jank byte-writes). Each scenario lists the profiles of the servers that do it, and that's where -like gets them from. The tests make sure each server's scenarios get its column of the compatibility matrix, apart from eval.no-code (clients always send some code) and the checks dialtone's and repartee's profiles skip for their languages, and that Who names every server a scenario lists. proof list shows which scenarios each server gets. As the flag package stops at the first scenario, a flag after one now gets a hint that flags go first instead of an unknown scenario error. --- README.md | 3 +- cmd/proof/main.go | 26 +++++++++-- cmd/proof/proxy_test.go | 2 +- cmd/proof/serve.go | 35 ++++++++++++--- cmd/proof/serve_test.go | 8 ++++ doc/design.md | 6 ++- doc/hacking.md | 21 ++++++--- doc/usage.md | 27 ++++++++--- internal/serve/matrix_test.go | 76 +++++++++++++++++++++++++------ internal/serve/scenarios.go | 85 +++++++++++++++++++++++++---------- 10 files changed, 228 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 44f56c0..88f3e53 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ Here it sends only the value of the last form (like Basilisp, jank and dialtone) and drops what the code prints to stderr (like Basilisp, dialtone and repartee). `proof list` shows all the scenarios, and every one of them is something a real server does (or something TCP can do to -the replies). +the replies). `-like jank` turns on all of jank's scenarios at once, and +`proof list` shows the other servers it can behave like. ## Documentation diff --git a/cmd/proof/main.go b/cmd/proof/main.go index b100a86..5c957e7 100644 --- a/cmd/proof/main.go +++ b/cmd/proof/main.go @@ -10,6 +10,7 @@ import ( "os" "os/signal" "path" + "strings" "syscall" "time" @@ -74,7 +75,7 @@ func printUsage(w io.Writer) { fmt.Fprint(w, "\nProxy flags:\n") proxyFlags(w, &proxyOptions{}).PrintDefaults() fmt.Fprint(w, "\nServe flags:\n") - serveFlags(w, &clientOptions{}).PrintDefaults() + serveFlags(w, &serveOptions{}).PrintDefaults() } type options struct { @@ -280,12 +281,31 @@ func filter(all []*check.Check, pattern string) ([]*check.Check, error) { return out, nil } +// The names in proof list take up this many columns, and its wrapped lines +// stay within lineWidth. +const nameWidth, lineWidth = 28, 100 + func list(w io.Writer) { for _, e := range append(catalog(), ruleEntries(checks.ClientRules())...) { - fmt.Fprintf(w, "%-28s %-4s %s\n", e.id, e.severity, e.title) + fmt.Fprintf(w, "%-*s %-4s %s\n", nameWidth, e.id, e.severity, e.title) } fmt.Fprint(w, "\nScenarios for proof serve:\n") for _, s := range serve.Scenarios() { - fmt.Fprintf(w, "%-28s %s (%s)\n", s.Name, s.Title, s.Who) + fmt.Fprintf(w, "%-*s %s (%s)\n", nameWidth, s.Name, s.Title, s.Who) + } + fmt.Fprint(w, "\nServers for proof serve -like, with their scenarios:\n") + for _, p := range serve.Presets() { + line := fmt.Sprintf("%-*s", nameWidth, p.Name) + for i, s := range p.Scenarios { + if i > 0 { + line += "," + if len(line)+len(" "+s+",") > lineWidth { + fmt.Fprintln(w, line) + line = strings.Repeat(" ", nameWidth) + } + } + line += " " + s + } + fmt.Fprintln(w, line) } } diff --git a/cmd/proof/proxy_test.go b/cmd/proof/proxy_test.go index 987a4de..c68c134 100644 --- a/cmd/proof/proxy_test.go +++ b/cmd/proof/proxy_test.go @@ -142,7 +142,7 @@ func TestProxyNeedsAServer(t *testing.T) { func TestListShowsEverything(t *testing.T) { var buf bytes.Buffer list(&buf) - for _, want := range []string{"eval.value", "wire.dict", "client.need-input", "split-output"} { + for _, want := range []string{"eval.value", "wire.dict", "client.need-input", "split-output", "clojure-clr"} { if !strings.Contains(buf.String(), want) { t.Errorf("list is missing %s:\n%s", want, buf.String()) } diff --git a/cmd/proof/serve.go b/cmd/proof/serve.go index f4bbd54..55f4bab 100644 --- a/cmd/proof/serve.go +++ b/cmd/proof/serve.go @@ -7,6 +7,8 @@ import ( "io" "os" "os/signal" + "path/filepath" + "slices" "strings" "syscall" "time" @@ -15,9 +17,15 @@ import ( "github.com/nrepl/proof/internal/serve" ) -func serveFlags(out io.Writer, o *clientOptions) *flag.FlagSet { +type serveOptions struct { + clientOptions + like string +} + +func serveFlags(out io.Writer, o *serveOptions) *flag.FlagSet { fs := flag.NewFlagSet("serve", flag.ContinueOnError) fs.SetOutput(out) + fs.StringVar(&o.like, "like", "", "behave like the server whose `profile` has this name (e.g. jank or profiles/jank.toml), along with any scenarios given") o.register(fs) return fs } @@ -35,12 +43,27 @@ func runServe(args []string) int { func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, listening func(addr string)) int { // The server logs to stderr from goroutines of its own. stderr = &syncWriter{w: stderr} - var o clientOptions + var o serveOptions fs := serveFlags(stderr, &o) if err := fs.Parse(args); err != nil { return 2 } - srv, err := serve.Listen(o.listen, version, fs.Args()) + scenarios, shown := fs.Args(), fs.Args() + // The flag package stops at the first scenario. + if i := slices.IndexFunc(scenarios, func(s string) bool { return strings.HasPrefix(s, "-") }); i >= 0 { + fmt.Fprintf(stderr, "proof: %s comes after a scenario, but flags have to go first\n", scenarios[i]) + return 2 + } + if o.like != "" { + name := strings.TrimSuffix(filepath.Base(o.like), ".toml") + var err error + if scenarios, err = serve.Like(name, scenarios); err != nil { + fmt.Fprintln(stderr, "proof:", err) + return 2 + } + shown = append([]string{"like " + name}, shown...) + } + srv, err := serve.Listen(o.listen, version, scenarios) if err != nil { fmt.Fprintln(stderr, "proof:", err) return 2 @@ -49,8 +72,8 @@ func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li go srv.Serve() started := time.Now() name := "proof serve" - if fs.NArg() > 0 { - name += " (" + strings.Join(fs.Args(), ", ") + ")" + if len(shown) > 0 { + name += " (" + strings.Join(shown, ", ") + ")" } fmt.Fprintf(stderr, "Running %s on %s. Connect your client and press Ctrl-C when it's done.\n", name, srv.Addr()) if listening != nil { @@ -60,5 +83,5 @@ func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li <-ctx.Done() fmt.Fprintln(stderr) r := report.Run{Proof: version, Server: "client traffic to " + name, Address: srv.Addr(), Started: started} - return gradeClients(stdout, stderr, r, srv.Stop(time.Second), o, nil) + return gradeClients(stdout, stderr, r, srv.Stop(time.Second), o.clientOptions, nil) } diff --git a/cmd/proof/serve_test.go b/cmd/proof/serve_test.go index d9b1649..96a9aba 100644 --- a/cmd/proof/serve_test.go +++ b/cmd/proof/serve_test.go @@ -19,6 +19,14 @@ func TestServeExitCodes(t *testing.T) { {"request without an id", nil, []string{"d2:op8:describee"}, 1, "a request without an id"}, {"no client", nil, nil, 3, "nothing to check"}, {"unknown scenario", []string{"no-such-scenario"}, nil, 2, `unknown scenario "no-such-scenario"`}, + {"like a server", []string{"-like", "jank", "byte-writes"}, []string{"d2:id1:12:op8:describee"}, 0, + "client traffic to proof serve (like jank, byte-writes)"}, + {"like an unknown server", []string{"-like", "no-such-server"}, nil, 2, `can't behave like "no-such-server"`}, + {"like a server, by its profile", []string{"-like", "profiles/jank.toml"}, []string{"d2:id1:12:op8:describee"}, 0, + "client traffic to proof serve (like jank)"}, + {"like a server, but not quite", []string{"-like", "jank", "ns-error"}, nil, 2, + "scenarios ns-fallback and ns-error don't go together"}, + {"flag after the scenarios", []string{"byte-writes", "-like", "jank"}, nil, 2, "-like comes after a scenario, but flags have to go first"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/doc/design.md b/doc/design.md index e3189b7..613e18d 100644 --- a/doc/design.md +++ b/doc/design.md @@ -168,8 +168,10 @@ TCP can do to any server's replies (e.g. deliver a message in pieces). That keeps the list down to the differences a client will actually run into, and the tests make sure it stays that way. proof runs its own checks against every scenario, and a scenario has to get the same -verdicts as the servers it names. All the scenarios of jank together get jank's -column of the matrix, for instance. +verdicts as the servers it names. `-like` turns on all the scenarios of a +server, and together they have to get that server's column of the matrix +(apart from `eval.no-code`, which no client sends, and the checks that +come down to the server's language). The grading works the other way around here. proof can't see what a client shows its users, so the client's own tests have to do that. Most diff --git a/doc/hacking.md b/doc/hacking.md index 63bceda..447c9ab 100644 --- a/doc/hacking.md +++ b/doc/hacking.md @@ -234,12 +234,16 @@ be something a real server does, and `Who` says which ones. To find out exactly what a server sends, connect to it and print the replies, the same way you would for a client rule. -If the scenario changes the verdict of some check, add it to the table in -`TestScenariosGetTheVerdictsOfTheirServers` (in `matrix_test.go`) with -the verdicts its servers get in the matrix, and to the columns of those -servers further down. Otherwise make sure it does what it says in -`TestScenarioShapes` (in `serve_test.go`). The tables in `doc/usage.md` -list all the scenarios as well. +List the profiles of the servers that do it as well, so `-like` turns it +on for them. A test makes sure `Who` names them. + +If the scenario changes the verdict of some check, add it to the table +in `TestScenariosGetTheVerdictsOfTheirServers` (in `matrix_test.go`) +with the verdicts its servers get in the matrix, and update the columns +of those servers in `TestPresetsGetTheColumnsOfTheirServers`. Otherwise +make sure it does what it says in `TestScenarioShapes` (in +`serve_test.go`). The tables in `doc/usage.md` list all the scenarios as +well. ## Adding a Server @@ -256,6 +260,11 @@ To add a server to the compatibility matrix: same way a user would. 4. Add it to the lists of profiles in the [README](../README.md) and in [Usage](usage.md). +5. Add it to `servers` in `internal/serve/scenarios.go`, and its profile + to the scenarios that cover how it differs from nREPL, so that + `proof serve -like` can behave like it. Add its column to + `TestPresetsGetTheColumnsOfTheirServers` too. If it does something no + scenario covers yet, see [Adding a Scenario](#adding-a-scenario). ## Links to Client Code diff --git a/doc/usage.md b/doc/usage.md index 3eb1dfc..4da4c04 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -149,7 +149,7 @@ Here are the options supported by `proof run`: | 3 | Some checks couldn't run at all. Usually this means that the server died during the run. | There are a couple of other commands as well. `proof list` shows all -the checks along with their severity (and the scenarios of +the checks along with their severity (and the scenarios and servers of `proof serve`), and `proof version` shows the version of proof. ## Running proof in CI @@ -336,6 +336,22 @@ the scenarios, and every one of them is something a real server does | `batched-writes` | Replies are held back and written together until the eval waits or ends | any server | | `hang-up` | The server closes the connection instead of answering an `eval` | any server that crashes | +`-like` turns on all the scenarios of a server at once. It takes the +server's profile (e.g. `jank` or `profiles/jank.toml`), and `proof list` +shows which scenarios each server gets: + +```shell +$ proof serve -listen 127.0.0.1:7888 -like jank +Running proof serve (like jank) on 127.0.0.1:7888. Connect your client and press Ctrl-C when it's done. +``` + +proof's own checks give `proof serve -like jank` the same results jank +gets in the compatibility matrix, and the same goes for the other +servers. There are a couple of exceptions. No scenario covers +`eval.no-code`, as clients always send some code, and the code is still +Clojure, even with `-like dialtone`. So the checks the profiles of +dialtone and repartee skip for their languages (e.g. `eval.ns`) pass. + Most scenarios change only how the replies look on the wire, not what a user should end up seeing. Evaluating `(println "hi") (+ 1 2)` should show `hi` and `3` with `split-output`, `byte-writes` or @@ -370,12 +386,13 @@ gets a read error. That's enough for CIDER to connect and work, and it's all you need for checking output, values, errors, input and interrupts. When you stop it, `proof serve` checks the requests your client sent, -just like `proof proxy` does, with the same report, options (except for -`-address`) and exit codes. A test suite can run it in CI like this: +just like `proof proxy` does, with the same report, exit codes and +`-listen`, `-json` and `-v` options. A test suite can run it in CI like +this: ```shell -for scenario in "" split-output last-value no-err byte-writes; do - proof serve -listen 127.0.0.1:7888 $scenario & +for args in "" split-output byte-writes -like=basilisp -like=jank; do + proof serve -listen 127.0.0.1:7888 $args & serve=$! until nc -z 127.0.0.1 7888; do kill -0 $serve || exit 2 diff --git a/internal/serve/matrix_test.go b/internal/serve/matrix_test.go index ac6b0ad..025ced3 100644 --- a/internal/serve/matrix_test.go +++ b/internal/serve/matrix_test.go @@ -2,6 +2,8 @@ package serve import ( "fmt" + "slices" + "strings" "testing" "time" @@ -71,19 +73,6 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) { {[]string{"unsorted-keys"}, map[string]check.Verdict{"wire.canonical": W}}, {[]string{"byte-writes"}, nil}, {[]string{"batched-writes"}, nil}, - // The columns of whole servers, except for eval.no-code, which no - // scenario covers as no client sends an eval without code. - {[]string{"no-op-echo", "socket-sessions", "shared-state", "no-session-closed", "any-session", "no-err", - "last-value", "ns-error", "no-stdin", "no-interrupt", "split-output"}, // Basilisp - map[string]check.Verdict{"op.unknown-echo": W, "session.across-connections": W, "session.isolated": F, - "session.close": F, "session.unknown": F, "session.closed": F, "eval.stderr": F, - "eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S}}, - {[]string{"no-close-op", "no-op-echo", "socket-sessions", "shared-state", "no-session-closed", "any-session", - "last-value", "ns-fallback", "read-line-throws", "no-interrupt", "unsorted-keys", "empty-messages", "error-with-done"}, // jank - map[string]check.Verdict{"describe.required-ops": F, "op.unknown-echo": W, "session.across-connections": W, - "session.isolated": F, "session.close": F, "session.unknown": F, "session.closed": F, - "eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, - "stdin.eof": S, "wire.canonical": W}}, } for _, c := range cases { t.Run(fmt.Sprint(c.scenarios), func(t *testing.T) { @@ -92,3 +81,64 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) { }) } } + +// A server's scenarios together get its column of the compatibility +// matrix. The columns leave out eval.no-code, as no client sends an eval +// without code, and the checks the profiles of dialtone and repartee skip +// for their languages. +func TestPresetsGetTheColumnsOfTheirServers(t *testing.T) { + F, W, S := check.Failed, check.Warned, check.Skipped + columns := map[string]map[string]check.Verdict{ + "clojure": {"stdin.eof": W}, + "babashka": nil, + "clojure-clr": {"op.unknown-echo": W, "session.across-connections": W, "session.isolated": F, + "session.unknown": F, "session.closed": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S}, + "basilisp": {"op.unknown-echo": W, "session.across-connections": W, "session.isolated": F, + "session.close": F, "session.unknown": F, "session.closed": F, "eval.stderr": F, + "eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S}, + "jank": {"describe.required-ops": F, "op.unknown-echo": W, "session.across-connections": W, + "session.isolated": F, "session.close": F, "session.unknown": F, "session.closed": F, + "eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, + "stdin.eof": S, "wire.canonical": W}, + "dialtone": {"op.unknown-echo": W, "session.close": F, "eval.stderr": F, "eval.multiple-forms": F}, + "repartee": {"op.unknown-echo": W, "session.close": F, "eval.stderr": F}, + } + for name, want := range columns { + t.Run(name, func(t *testing.T) { + t.Parallel() + scenarios, err := Like(name, nil) + if err != nil { + t.Fatal(err) + } + checktest.Verdicts(t, runChecks(t, scenarios...), want) + }) + } + for _, p := range Presets() { + if _, ok := columns[p.Name]; !ok { + t.Errorf("no column for %s", p.Name) + } + } +} + +// Each server is named by its profile, and the scenarios it has say they're +// what it does. +func TestPresetsMatchTheScenarios(t *testing.T) { + for _, p := range Presets() { + prof, err := profile.Load("../../profiles/" + p.Name + ".toml") + if err != nil { + t.Errorf("%s isn't the name of a profile: %v", p.Name, err) + continue + } + // e.g. "ClojureCLR (clr.tools.nrepl 0.1.2-alpha2)" says ClojureCLR, + // but the profile of nREPL itself is named after Clojure. + server, _, _ := strings.Cut(prof.Name, " (") + if p.Name == "clojure" { + server = "nREPL 1.7.0" + } + for _, s := range Scenarios() { + if slices.Contains(p.Scenarios, s.Name) && !strings.Contains(s.Who, server) { + t.Errorf("%s has %s, which doesn't say %s does it", p.Name, s.Name, server) + } + } + } +} diff --git a/internal/serve/scenarios.go b/internal/serve/scenarios.go index 23e1924..834f8e9 100644 --- a/internal/serve/scenarios.go +++ b/internal/serve/scenarios.go @@ -39,60 +39,62 @@ const ( throwOnRead ) -// scenario is a Scenario and the behavior it sets. +// scenario is a Scenario, the behavior it sets, and the profiles of the +// servers that do it. type scenario struct { Scenario - set func(*behavior) + set func(*behavior) + servers []string } var catalog = []scenario{ {Scenario{"split-output", "Output comes one character per message", "nREPL splits long output, and Basilisp sends println's newline on its own"}, - func(b *behavior) { b.splitOutput = true }}, + func(b *behavior) { b.splitOutput = true }, []string{"basilisp"}}, {Scenario{"empty-messages", "Replies to eval include messages with nothing but id and session", "jank"}, - func(b *behavior) { b.emptyMessages = true }}, + func(b *behavior) { b.emptyMessages = true }, []string{"jank"}}, {Scenario{"last-value", "Only the value of the last form is sent", "Basilisp, jank, dialtone"}, - func(b *behavior) { b.lastValue = true }}, + func(b *behavior) { b.lastValue = true }, []string{"basilisp", "jank", "dialtone"}}, {Scenario{"no-err", "What the code prints to stderr never reaches the client", "Basilisp, dialtone, repartee"}, - func(b *behavior) { b.noErr = true }}, + func(b *behavior) { b.noErr = true }, []string{"basilisp", "dialtone", "repartee"}}, {Scenario{"error-with-done", "eval-error comes in the same message as done", "jank"}, - func(b *behavior) { b.errorWithDone = true }}, + func(b *behavior) { b.errorWithDone = true }, []string{"jank"}}, {Scenario{"no-op-echo", "Replies to unknown ops don't say which op it was", "ClojureCLR, Basilisp, jank, dialtone, repartee"}, - func(b *behavior) { b.noOpEcho = true }}, + func(b *behavior) { b.noOpEcho = true }, []string{"clojure-clr", "basilisp", "jank", "dialtone", "repartee"}}, {Scenario{"no-close-op", "describe doesn't list close, even though close works", "jank"}, - func(b *behavior) { b.noCloseOp = true }}, + func(b *behavior) { b.noCloseOp = true }, []string{"jank"}}, {Scenario{"no-interrupt", "There's no interrupt op", "ClojureCLR, Basilisp, jank"}, - func(b *behavior) { b.noInterrupt = true }}, + func(b *behavior) { b.noInterrupt = true }, []string{"clojure-clr", "basilisp", "jank"}}, {Scenario{"no-stdin", "There's no stdin op, and reading input gets nil right away", "ClojureCLR and Basilisp started in the background, as they read their own stdin, where Basilisp gets an empty string"}, - func(b *behavior) { b.stdin = endOfInput }}, + func(b *behavior) { b.stdin = endOfInput }, []string{"clojure-clr", "basilisp"}}, {Scenario{"read-line-throws", "There's no stdin op, and reading input throws", "jank"}, - func(b *behavior) { b.stdin = throwOnRead }}, + func(b *behavior) { b.stdin = throwOnRead }, []string{"jank"}}, {Scenario{"string-versions", "versions.proof is a plain string rather than a dict", "Babashka for versions.babashka, ClojureCLR for versions.clojure.tools.nrepl"}, - func(b *behavior) { b.stringVersions = true }}, + func(b *behavior) { b.stringVersions = true }, []string{"babashka", "clojure-clr"}}, {Scenario{"no-session-closed", "close replies with done alone, without session-closed", "Basilisp, jank, dialtone, repartee"}, - func(b *behavior) { b.noSessionClosed = true }}, + func(b *behavior) { b.noSessionClosed = true }, []string{"basilisp", "jank", "dialtone", "repartee"}}, {Scenario{"shared-state", "Sessions on the same connection share *1, *e and the current namespace", "ClojureCLR, Basilisp, jank"}, - func(b *behavior) { b.sharedState = true }}, + func(b *behavior) { b.sharedState = true }, []string{"clojure-clr", "basilisp", "jank"}}, {Scenario{"socket-sessions", "A session only exists on the connection that cloned it", "ClojureCLR, Basilisp, jank"}, - func(b *behavior) { b.socketSessions = true }}, + func(b *behavior) { b.socketSessions = true }, []string{"clojure-clr", "basilisp", "jank"}}, {Scenario{"any-session", "Requests for sessions that don't exist run in a new session", "ClojureCLR, Basilisp, jank"}, - func(b *behavior) { b.anySession = true }}, + func(b *behavior) { b.anySession = true }, []string{"clojure-clr", "basilisp", "jank"}}, {Scenario{"ns-fallback", "An eval in a namespace that doesn't exist runs in the current one", "jank"}, - func(b *behavior) { b.nsFallback = true }}, + func(b *behavior) { b.nsFallback = true }, []string{"jank"}}, {Scenario{"ns-error", "An eval in a namespace that doesn't exist fails without namespace-not-found", "Basilisp"}, - func(b *behavior) { b.nsError = true }}, + func(b *behavior) { b.nsError = true }, []string{"basilisp"}}, {Scenario{"eof-error", "Reading past the end of input fails instead of returning nil", "nREPL 1.7.0"}, - func(b *behavior) { b.eofError = true }}, + func(b *behavior) { b.eofError = true }, []string{"clojure"}}, {Scenario{"unsorted-keys", "The keys of reply dicts aren't sorted", "jank"}, - func(b *behavior) { b.unsortedKeys = true }}, + func(b *behavior) { b.unsortedKeys = true }, []string{"jank"}}, {Scenario{"byte-writes", "Replies are written a byte at a time", "any server, as TCP can deliver a message in pieces"}, - func(b *behavior) { b.byteWrites = true }}, + func(b *behavior) { b.byteWrites = true }, nil}, {Scenario{"batched-writes", "Replies are held back and written together until the eval waits or ends", "any server, as TCP can deliver several messages together"}, - func(b *behavior) { b.batchedWrites = true }}, + func(b *behavior) { b.batchedWrites = true }, nil}, {Scenario{"hang-up", "The server closes the connection instead of answering an eval", "any server that crashes"}, - func(b *behavior) { b.hangUp = true }}, + func(b *behavior) { b.hangUp = true }, nil}, } // Scenarios lists every scenario. @@ -104,6 +106,41 @@ func Scenarios() []Scenario { return s } +// Preset is a real server as the scenarios that make proof serve behave +// like it. Name is the name of the server's profile. +type Preset struct { + Name string + Scenarios []string +} + +// servers are the profiles of the servers with scenarios. +var servers = []string{"clojure", "babashka", "clojure-clr", "basilisp", "jank", "dialtone", "repartee"} + +// Presets lists every server proof serve can behave like. +func Presets() []Preset { + presets := make([]Preset, len(servers)) + for i, name := range servers { + presets[i].Name = name + for _, c := range catalog { + if slices.Contains(c.servers, name) { + presets[i].Scenarios = append(presets[i].Scenarios, c.Name) + } + } + } + return presets +} + +// Like returns the scenarios of the server whose profile has the given +// name, followed by the extra ones. +func Like(name string, extra []string) ([]string, error) { + presets := Presets() + i := slices.IndexFunc(presets, func(p Preset) bool { return p.Name == name }) + if i < 0 { + return nil, fmt.Errorf("proof serve can't behave like %q (proof list shows the servers it can)", name) + } + return append(presets[i].Scenarios, extra...), nil +} + // conflicts are scenarios that can't be combined. var conflicts = [][2]string{{"ns-fallback", "ns-error"}, {"byte-writes", "batched-writes"}, {"no-stdin", "read-line-throws"}}