Let proof serve behave like a whole server

README.md

@@ -103,7 +103,8 @@ Here it sends only the value of the last form (like Basilisp, jank and
 dialtone) and drops what the code prints to stderr (like Basilisp,
 dialtone and repartee). `proof list` shows all the scenarios, and every
 one of them is something a real server does (or something TCP can do to
-the replies).
+the replies). `-like jank` turns on all of jank's scenarios at once, and
+`proof list` shows the other servers it can behave like.
 
 ## Documentation
 

cmd/proof/main.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"strings"
 	"syscall"
 	"time"
 
@@ -74,7 +75,7 @@ func printUsage(w io.Writer) {
 	fmt.Fprint(w, "\nProxy flags:\n")
 	proxyFlags(w, &proxyOptions{}).PrintDefaults()
 	fmt.Fprint(w, "\nServe flags:\n")
-	serveFlags(w, &clientOptions{}).PrintDefaults()
+	serveFlags(w, &serveOptions{}).PrintDefaults()
 }
 
 type options struct {
@@ -280,12 +281,31 @@ func filter(all []*check.Check, pattern string) ([]*check.Check, error) {
 	return out, nil
 }
 
+// The names in proof list take up this many columns, and its wrapped lines
+// stay within lineWidth.
+const nameWidth, lineWidth = 28, 100
+
 func list(w io.Writer) {
 	for _, e := range append(catalog(), ruleEntries(checks.ClientRules())...) {
-		fmt.Fprintf(w, "%-28s %-4s %s\n", e.id, e.severity, e.title)
+		fmt.Fprintf(w, "%-*s %-4s %s\n", nameWidth, e.id, e.severity, e.title)
 	}
 	fmt.Fprint(w, "\nScenarios for proof serve:\n")
 	for _, s := range serve.Scenarios() {
-		fmt.Fprintf(w, "%-28s %s (%s)\n", s.Name, s.Title, s.Who)
+		fmt.Fprintf(w, "%-*s %s (%s)\n", nameWidth, s.Name, s.Title, s.Who)
+	}
+	fmt.Fprint(w, "\nServers for proof serve -like, with their scenarios:\n")
+	for _, p := range serve.Presets() {
+		line := fmt.Sprintf("%-*s", nameWidth, p.Name)
+		for i, s := range p.Scenarios {
+			if i > 0 {
+				line += ","
+				if len(line)+len(" "+s+",") > lineWidth {
+					fmt.Fprintln(w, line)
+					line = strings.Repeat(" ", nameWidth)
+				}
+			}
+			line += " " + s
+		}
+		fmt.Fprintln(w, line)
 	}
 }

cmd/proof/proxy_test.go

@@ -142,7 +142,7 @@ func TestProxyNeedsAServer(t *testing.T) {
 func TestListShowsEverything(t *testing.T) {
 	var buf bytes.Buffer
 	list(&buf)
-	for _, want := range []string{"eval.value", "wire.dict", "client.need-input", "split-output"} {
+	for _, want := range []string{"eval.value", "wire.dict", "client.need-input", "split-output", "clojure-clr"} {
 		if !strings.Contains(buf.String(), want) {
 			t.Errorf("list is missing %s:\n%s", want, buf.String())
 		}

cmd/proof/serve.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"os"
 	"os/signal"
+	"path/filepath"
+	"slices"
 	"strings"
 	"syscall"
 	"time"
@@ -15,9 +17,15 @@ import (
 	"github.com/nrepl/proof/internal/serve"
 )
 
-func serveFlags(out io.Writer, o *clientOptions) *flag.FlagSet {
+type serveOptions struct {
+	clientOptions
+	like string
+}
+
+func serveFlags(out io.Writer, o *serveOptions) *flag.FlagSet {
 	fs := flag.NewFlagSet("serve", flag.ContinueOnError)
 	fs.SetOutput(out)
+	fs.StringVar(&o.like, "like", "", "behave like the server whose `profile` has this name (e.g. jank or profiles/jank.toml), along with any scenarios given")
 	o.register(fs)
 	return fs
 }
@@ -35,12 +43,27 @@ func runServe(args []string) int {
 func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, listening func(addr string)) int {
 	// The server logs to stderr from goroutines of its own.
 	stderr = &syncWriter{w: stderr}
-	var o clientOptions
+	var o serveOptions
 	fs := serveFlags(stderr, &o)
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
-	srv, err := serve.Listen(o.listen, version, fs.Args())
+	scenarios, shown := fs.Args(), fs.Args()
+	// The flag package stops at the first scenario.
+	if i := slices.IndexFunc(scenarios, func(s string) bool { return strings.HasPrefix(s, "-") }); i >= 0 {
+		fmt.Fprintf(stderr, "proof: %s comes after a scenario, but flags have to go first\n", scenarios[i])
+		return 2
+	}
+	if o.like != "" {
+		name := strings.TrimSuffix(filepath.Base(o.like), ".toml")
+		var err error
+		if scenarios, err = serve.Like(name, scenarios); err != nil {
+			fmt.Fprintln(stderr, "proof:", err)
+			return 2
+		}
+		shown = append([]string{"like " + name}, shown...)
+	}
+	srv, err := serve.Listen(o.listen, version, scenarios)
 	if err != nil {
 		fmt.Fprintln(stderr, "proof:", err)
 		return 2
@@ -49,8 +72,8 @@ func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li
 	go srv.Serve()
 	started := time.Now()
 	name := "proof serve"
-	if fs.NArg() > 0 {
-		name += " (" + strings.Join(fs.Args(), ", ") + ")"
+	if len(shown) > 0 {
+		name += " (" + strings.Join(shown, ", ") + ")"
 	}
 	fmt.Fprintf(stderr, "Running %s on %s. Connect your client and press Ctrl-C when it's done.\n", name, srv.Addr())
 	if listening != nil {
@@ -60,5 +83,5 @@ func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li
 	<-ctx.Done()
 	fmt.Fprintln(stderr)
 	r := report.Run{Proof: version, Server: "client traffic to " + name, Address: srv.Addr(), Started: started}
-	return gradeClients(stdout, stderr, r, srv.Stop(time.Second), o, nil)
+	return gradeClients(stdout, stderr, r, srv.Stop(time.Second), o.clientOptions, nil)
 }

cmd/proof/serve_test.go

@@ -19,6 +19,14 @@ func TestServeExitCodes(t *testing.T) {
 		{"request without an id", nil, []string{"d2:op8:describee"}, 1, "a request without an id"},
 		{"no client", nil, nil, 3, "nothing to check"},
 		{"unknown scenario", []string{"no-such-scenario"}, nil, 2, `unknown scenario "no-such-scenario"`},
+		{"like a server", []string{"-like", "jank", "byte-writes"}, []string{"d2:id1:12:op8:describee"}, 0,
+			"client traffic to proof serve (like jank, byte-writes)"},
+		{"like an unknown server", []string{"-like", "no-such-server"}, nil, 2, `can't behave like "no-such-server"`},
+		{"like a server, by its profile", []string{"-like", "profiles/jank.toml"}, []string{"d2:id1:12:op8:describee"}, 0,
+			"client traffic to proof serve (like jank)"},
+		{"like a server, but not quite", []string{"-like", "jank", "ns-error"}, nil, 2,
+			"scenarios ns-fallback and ns-error don't go together"},
+		{"flag after the scenarios", []string{"byte-writes", "-like", "jank"}, nil, 2, "-like comes after a scenario, but flags have to go first"},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {

doc/design.md

@@ -168,8 +168,10 @@ TCP can do to any server's replies (e.g. deliver a message in pieces).
 That keeps the list down to the differences a client will actually run
 into, and the tests make sure it stays that way. proof runs its own
 checks against every scenario, and a scenario has to get the same
-verdicts as the servers it names. All the scenarios of jank together get jank's
-column of the matrix, for instance.
+verdicts as the servers it names. `-like` turns on all the scenarios of a
+server, and together they have to get that server's column of the matrix
+(apart from `eval.no-code`, which no client sends, and the checks that
+come down to the server's language).
 
 The grading works the other way around here. proof can't see what a
 client shows its users, so the client's own tests have to do that. Most

doc/hacking.md

@@ -234,12 +234,16 @@ be something a real server does, and `Who` says which ones. To find out
 exactly what a server sends, connect to it and print the replies, the
 same way you would for a client rule.
 
-If the scenario changes the verdict of some check, add it to the table in
-`TestScenariosGetTheVerdictsOfTheirServers` (in `matrix_test.go`) with
-the verdicts its servers get in the matrix, and to the columns of those
-servers further down. Otherwise make sure it does what it says in
-`TestScenarioShapes` (in `serve_test.go`). The tables in `doc/usage.md`
-list all the scenarios as well.
+List the profiles of the servers that do it as well, so `-like` turns it
+on for them. A test makes sure `Who` names them.
+
+If the scenario changes the verdict of some check, add it to the table
+in `TestScenariosGetTheVerdictsOfTheirServers` (in `matrix_test.go`)
+with the verdicts its servers get in the matrix, and update the columns
+of those servers in `TestPresetsGetTheColumnsOfTheirServers`. Otherwise
+make sure it does what it says in `TestScenarioShapes` (in
+`serve_test.go`). The tables in `doc/usage.md` list all the scenarios as
+well.
 
 ## Adding a Server
 
@@ -256,6 +260,11 @@ To add a server to the compatibility matrix:
    same way a user would.
 4. Add it to the lists of profiles in the [README](../README.md) and in
    [Usage](usage.md).
+5. Add it to `servers` in `internal/serve/scenarios.go`, and its profile
+   to the scenarios that cover how it differs from nREPL, so that
+   `proof serve -like` can behave like it. Add its column to
+   `TestPresetsGetTheColumnsOfTheirServers` too. If it does something no
+   scenario covers yet, see [Adding a Scenario](#adding-a-scenario).
 
 ## Links to Client Code
 

doc/usage.md

@@ -149,7 +149,7 @@ Here are the options supported by `proof run`:
 | 3 | Some checks couldn't run at all. Usually this means that the server died during the run. |
 
 There are a couple of other commands as well. `proof list` shows all
-the checks along with their severity (and the scenarios of
+the checks along with their severity (and the scenarios and servers of
 `proof serve`), and `proof version` shows the version of proof.
 
 ## Running proof in CI
@@ -320,9 +320,10 @@ the scenarios, and every one of them is something a real server does
 | `error-with-done` | `eval-error` comes in the same message as `done` | jank |
 | `no-op-echo` | Replies to unknown ops don't say which op it was | ClojureCLR, Basilisp, jank, dialtone, repartee |
 | `no-close-op` | `describe` doesn't list `close`, even though `close` works | jank |
-| `no-interrupt` | There's no `interrupt` op | Basilisp, jank |
-| `no-stdin` | There's no `stdin` op, and reading input gets an empty string right away | Basilisp |
-| `string-versions` | `versions.proof` is a plain string rather than a dict | Babashka, for `versions.babashka` |
+| `no-interrupt` | There's no `interrupt` op | ClojureCLR, Basilisp, jank |
+| `no-stdin` | There's no `stdin` op, and reading input gets `nil` right away | ClojureCLR and Basilisp started in the background, as they read their own stdin, where Basilisp gets an empty string |
+| `read-line-throws` | There's no `stdin` op, and reading input throws | jank |
+| `string-versions` | `versions.proof` is a plain string rather than a dict | Babashka for `versions.babashka`, ClojureCLR for `versions.clojure.tools.nrepl` |
 | `no-session-closed` | `close` replies with `done` alone | Basilisp, jank, dialtone, repartee |
 | `shared-state` | Sessions on the same connection share `*1`, `*e` and the current namespace | ClojureCLR, Basilisp, jank |
 | `socket-sessions` | A session only exists on the connection that cloned it | ClojureCLR, Basilisp, jank |
@@ -335,6 +336,22 @@ the scenarios, and every one of them is something a real server does
 | `batched-writes` | Replies are held back and written together until the eval waits or ends | any server |
 | `hang-up` | The server closes the connection instead of answering an `eval` | any server that crashes |
 
+`-like` turns on all the scenarios of a server at once. It takes the
+server's profile (e.g. `jank` or `profiles/jank.toml`), and `proof list`
+shows which scenarios each server gets:
+
+```shell
+$ proof serve -listen 127.0.0.1:7888 -like jank
+Running proof serve (like jank) on 127.0.0.1:7888. Connect your client and press Ctrl-C when it's done.
+```
+
+proof's own checks give `proof serve -like jank` the same results jank
+gets in the compatibility matrix, and the same goes for the other
+servers. There are a couple of exceptions. No scenario covers
+`eval.no-code`, as clients always send some code, and the code is still
+Clojure, even with `-like dialtone`. So the checks the profiles of
+dialtone and repartee skip for their languages (e.g. `eval.ns`) pass.
+
 Most scenarios change only how the replies look on the wire, not what a
 user should end up seeing. Evaluating `(println "hi") (+ 1 2)` should
 show `hi` and `3` with `split-output`, `byte-writes` or
@@ -369,12 +386,13 @@ gets a read error. That's enough for CIDER to connect and work, and it's
 all you need for checking output, values, errors, input and interrupts.
 
 When you stop it, `proof serve` checks the requests your client sent,
-just like `proof proxy` does, with the same report, options (except for
-`-address`) and exit codes. A test suite can run it in CI like this:
+just like `proof proxy` does, with the same report, exit codes and
+`-listen`, `-json` and `-v` options. A test suite can run it in CI like
+this:
 
 ```shell
-for scenario in "" split-output last-value no-err byte-writes; do
-  proof serve -listen 127.0.0.1:7888 $scenario &
+for args in "" split-output byte-writes -like=basilisp -like=jank; do
+  proof serve -listen 127.0.0.1:7888 $args &
   serve=$!
   until nc -z 127.0.0.1 7888; do
     kill -0 $serve || exit 2

internal/serve/matrix_test.go

@@ -2,6 +2,8 @@ package serve
 
 import (
 	"fmt"
+	"slices"
+	"strings"
 	"testing"
 	"time"
 
@@ -59,6 +61,7 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) {
 		{[]string{"no-close-op"}, map[string]check.Verdict{"describe.required-ops": F}},
 		{[]string{"no-interrupt"}, nil},
 		{[]string{"no-stdin"}, noStdin},
+		{[]string{"read-line-throws"}, noStdin},
 		{[]string{"string-versions"}, nil},
 		{[]string{"no-session-closed"}, map[string]check.Verdict{"session.close": F}},
 		{[]string{"shared-state"}, map[string]check.Verdict{"session.isolated": F}},
@@ -70,19 +73,6 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) {
 		{[]string{"unsorted-keys"}, map[string]check.Verdict{"wire.canonical": W}},
 		{[]string{"byte-writes"}, nil},
 		{[]string{"batched-writes"}, nil},
-		// The columns of whole servers, except for eval.no-code, which no
-		// scenario covers as no client sends an eval without code.
-		{[]string{"no-op-echo", "socket-sessions", "shared-state", "no-session-closed", "any-session", "no-err",
-			"last-value", "ns-error", "no-stdin", "no-interrupt", "split-output"}, // Basilisp
-			map[string]check.Verdict{"op.unknown-echo": W, "session.across-connections": W, "session.isolated": F,
-				"session.close": F, "session.unknown": F, "session.closed": F, "eval.stderr": F,
-				"eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S}},
-		{[]string{"no-close-op", "no-op-echo", "socket-sessions", "shared-state", "no-session-closed", "any-session",
-			"last-value", "ns-fallback", "no-stdin", "no-interrupt", "unsorted-keys", "empty-messages", "error-with-done"}, // jank
-			map[string]check.Verdict{"describe.required-ops": F, "op.unknown-echo": W, "session.across-connections": W,
-				"session.isolated": F, "session.close": F, "session.unknown": F, "session.closed": F,
-				"eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S,
-				"stdin.eof": S, "wire.canonical": W}},
 	}
 	for _, c := range cases {
 		t.Run(fmt.Sprint(c.scenarios), func(t *testing.T) {
@@ -91,3 +81,64 @@ func TestScenariosGetTheVerdictsOfTheirServers(t *testing.T) {
 		})
 	}
 }
+
+// A server's scenarios together get its column of the compatibility
+// matrix. The columns leave out eval.no-code, as no client sends an eval
+// without code, and the checks the profiles of dialtone and repartee skip
+// for their languages.
+func TestPresetsGetTheColumnsOfTheirServers(t *testing.T) {
+	F, W, S := check.Failed, check.Warned, check.Skipped
+	columns := map[string]map[string]check.Verdict{
+		"clojure":  {"stdin.eof": W},
+		"babashka": nil,
+		"clojure-clr": {"op.unknown-echo": W, "session.across-connections": W, "session.isolated": F,
+			"session.unknown": F, "session.closed": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S},
+		"basilisp": {"op.unknown-echo": W, "session.across-connections": W, "session.isolated": F,
+			"session.close": F, "session.unknown": F, "session.closed": F, "eval.stderr": F,
+			"eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S, "stdin.eof": S},
+		"jank": {"describe.required-ops": F, "op.unknown-echo": W, "session.across-connections": W,
+			"session.isolated": F, "session.close": F, "session.unknown": F, "session.closed": F,
+			"eval.multiple-forms": F, "eval.unknown-ns": F, "stdin.need-input": S, "stdin.roundtrip": S,
+			"stdin.eof": S, "wire.canonical": W},
+		"dialtone": {"op.unknown-echo": W, "session.close": F, "eval.stderr": F, "eval.multiple-forms": F},
+		"repartee": {"op.unknown-echo": W, "session.close": F, "eval.stderr": F},
+	}
+	for name, want := range columns {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			scenarios, err := Like(name, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			checktest.Verdicts(t, runChecks(t, scenarios...), want)
+		})
+	}
+	for _, p := range Presets() {
+		if _, ok := columns[p.Name]; !ok {
+			t.Errorf("no column for %s", p.Name)
+		}
+	}
+}
+
+// Each server is named by its profile, and the scenarios it has say they're
+// what it does.
+func TestPresetsMatchTheScenarios(t *testing.T) {
+	for _, p := range Presets() {
+		prof, err := profile.Load("../../profiles/" + p.Name + ".toml")
+		if err != nil {
+			t.Errorf("%s isn't the name of a profile: %v", p.Name, err)
+			continue
+		}
+		// e.g. "ClojureCLR (clr.tools.nrepl 0.1.2-alpha2)" says ClojureCLR,
+		// but the profile of nREPL itself is named after Clojure.
+		server, _, _ := strings.Cut(prof.Name, " (")
+		if p.Name == "clojure" {
+			server = "nREPL 1.7.0"
+		}
+		for _, s := range Scenarios() {
+			if slices.Contains(p.Scenarios, s.Name) && !strings.Contains(s.Who, server) {
+				t.Errorf("%s has %s, which doesn't say %s does it", p.Name, s.Name, server)
+			}
+		}
+	}
+}