Add proof serve for testing clients against other servers

README.md

@@ -9,7 +9,8 @@ people use (CIDER, Calva, Conjure, vim-fireplace, REPLy and so on). It
 talks to the server over a socket like any client would, runs a set of
 checks against it and tells you what's broken and which clients it breaks.
 It can also check the other side of the conversation, i.e. the requests
-an nREPL client sends.
+an nREPL client sends, and stand in for other servers in a client's
+tests.
 
 The [nREPL protocol spec](https://spec.nrepl.org) is still a draft and in
 a few places it disagrees with what clients actually do. When that
@@ -89,6 +90,21 @@ and that sessions get closed in the end. Here a failure means that some
 server won't work properly with your client, and the report links to the
 server code in question.
 
+To see how your client deals with the replies of different servers, run
+its tests against `proof serve`. It's a small nREPL server that behaves
+like nREPL itself, unless you ask it to behave like some other server in
+one particular way:
+
+```shell
+$ proof serve -listen 127.0.0.1:7888 last-value no-err
+```
+
+Here it sends only the value of the last form (like Basilisp, jank and
+dialtone) and drops what the code prints to stderr (like Basilisp,
+dialtone and repartee). `proof list` shows all the scenarios, and every
+one of them is something a real server does (or something TCP can do to
+the replies).
+
 ## Documentation
 
 - [Usage](doc/usage.md) - checking your server, reading the report,
@@ -106,8 +122,8 @@ server code in question.
 
 proof is still in its early days. Right now it covers the core of the
 protocol (`describe`, unknown ops, `eval`, sessions, `stdin` and the wire format),
-along with the requests clients send, and `proof list` will show you all
-the checks.
+along with the requests clients send and the server differences clients
+have to deal with, and `proof list` will show you all the checks.
 
 Here's what's coming next:
 
@@ -116,9 +132,6 @@ Here's what's coming next:
   clients disconnecting in the middle of an evaluation)
 - replaying what real clients send (e.g. when CIDER or Calva connect to a
   server) as client profiles
-- a server that misbehaves on purpose (late output, output split into
-  many messages and so on), so client test suites can check how their
-  client deals with replies
 - publishing the compatibility matrix somewhere nicer than a CI job
   summary
 

cmd/proof/main.go

@@ -1,5 +1,5 @@
 // Command proof checks an nREPL server's compatibility with the clients
-// people actually use, and what a client sends to a server.
+// people actually use, and checks clients against servers.
 package main
 
 import (
@@ -17,6 +17,7 @@ import (
 	"github.com/nrepl/proof/internal/checks"
 	"github.com/nrepl/proof/internal/profile"
 	"github.com/nrepl/proof/internal/report"
+	"github.com/nrepl/proof/internal/serve"
 	"github.com/nrepl/proof/internal/server"
 )
 
@@ -25,17 +26,18 @@ const version = "0.1.0-dev"
 const usage = `proof checks an nREPL server's compatibility with existing clients.
 
 Usage:
-  proof run [flags] PROFILE       run the checks against the server a profile describes
-  proof proxy [flags] [PROFILE]   check what a client sends to a server, by sitting between them
-  proof matrix REPORT...          build a Markdown compatibility matrix from JSON reports
-  proof list                      list every check and rule
+  proof run [flags] PROFILE           run the checks against the server a profile describes
+  proof proxy [flags] [PROFILE]       check what a client sends to a server, by sitting between them
+  proof serve [flags] [SCENARIO...]   be a server for a client's tests, acting like other servers where asked
+  proof matrix REPORT...              build a Markdown compatibility matrix from JSON reports
+  proof list                          list every check, rule and scenario
   proof version
 
 The exit status of run is 0 when everything passed (or failed as the
 profile expects), 1 when the server failed checks, 2 when proof couldn't
 start (bad flags or profile, or the server didn't come up), and 3 when some
-checks couldn't run at all. The same goes for proxy, where 1 means the
-client failed rules and 3 means no client sent anything.
+checks couldn't run at all. The same goes for proxy and serve, where 1
+means the client failed rules and 3 means no client sent anything.
 
 Run flags:
 `
@@ -50,6 +52,8 @@ func main() {
 		os.Exit(run(os.Args[2:]))
 	case "proxy":
 		os.Exit(runProxy(os.Args[2:]))
+	case "serve":
+		os.Exit(runServe(os.Args[2:]))
 	case "matrix":
 		os.Exit(matrix(os.Args[2:]))
 	case "list":
@@ -69,6 +73,8 @@ func printUsage(w io.Writer) {
 	runFlags(w).PrintDefaults()
 	fmt.Fprint(w, "\nProxy flags:\n")
 	proxyFlags(w, &proxyOptions{}).PrintDefaults()
+	fmt.Fprint(w, "\nServe flags:\n")
+	serveFlags(w, &clientOptions{}).PrintDefaults()
 }
 
 type options struct {
@@ -278,4 +284,8 @@ func list(w io.Writer) {
 	for _, e := range append(catalog(), ruleEntries(checks.ClientRules())...) {
 		fmt.Fprintf(w, "%-28s %-4s %s\n", e.id, e.severity, e.title)
 	}
+	fmt.Fprint(w, "\nScenarios for proof serve:\n")
+	for _, s := range serve.Scenarios() {
+		fmt.Fprintf(w, "%-28s %s (%s)\n", s.Name, s.Title, s.Who)
+	}
 }

cmd/proof/proxy.go

@@ -19,23 +19,39 @@ import (
 	"github.com/nrepl/proof/internal/server"
 )
 
-type proxyOptions struct {
-	address string
+// clientOptions are the options of the commands that check clients.
+type clientOptions struct {
 	listen  string
 	json    string
 	verbose bool
 }
 
+func (o *clientOptions) register(fs *flag.FlagSet) {
+	fs.StringVar(&o.listen, "listen", "127.0.0.1:0", "accept clients on `host:port` (port 0 picks a free one)")
+	fs.StringVar(&o.json, "json", "", "also write a JSON report to `file`")
+	fs.BoolVar(&o.verbose, "v", false, "show every message the client and the server exchanged")
+}
+
+type proxyOptions struct {
+	clientOptions
+	address string
+}
+
 func proxyFlags(out io.Writer, o *proxyOptions) *flag.FlagSet {
 	fs := flag.NewFlagSet("proxy", flag.ContinueOnError)
 	fs.SetOutput(out)
 	fs.StringVar(&o.address, "address", "", "forward clients to a server already running at `host:port` instead of launching the one the profile describes")
-	fs.StringVar(&o.listen, "listen", "127.0.0.1:0", "accept clients on `host:port` (port 0 picks a free one)")
-	fs.StringVar(&o.json, "json", "", "also write a JSON report to `file`")
-	fs.BoolVar(&o.verbose, "v", false, "show every message the client and the server exchanged")
+	o.register(fs)
 	return fs
 }
 
+// logTo writes log messages to w, one per line.
+func logTo(w io.Writer) func(format string, args ...any) {
+	return func(format string, args ...any) {
+		fmt.Fprintf(w, format+"\n", args...)
+	}
+}
+
 // runProxy sits between a client and a server until it's interrupted, and
 // then grades everything the client sent.
 func runProxy(args []string) int {
@@ -95,9 +111,7 @@ func proxyUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li
 		fmt.Fprintln(stderr, "proof:", err)
 		return 2
 	}
-	px.Logf = func(format string, args ...any) {
-		fmt.Fprintf(stderr, format+"\n", args...)
-	}
+	px.Logf = logTo(stderr)
 	go px.Serve()
 	started := time.Now()
 	fmt.Fprintf(stderr, "Forwarding %s to %s. Connect your client to %s and press Ctrl-C when it's done.\n",
@@ -108,19 +122,19 @@ func proxyUntil(ctx context.Context, args []string, stdout, stderr io.Writer, li
 
 	<-ctx.Done()
 	fmt.Fprintln(stderr)
-	traffic := px.Stop(time.Second)
+	r := report.Run{Proof: version, Server: "client traffic to " + name, Address: upstream, Started: started}
+	return gradeClients(stdout, stderr, r, px.Stop(time.Second), o.clientOptions, srv)
+}
+
+// gradeClients grades the requests in traffic, reports on them, and
+// returns the exit status. srv is the server proof started, if any.
+func gradeClients(stdout, stderr io.Writer, r report.Run, traffic []check.Traffic, o clientOptions, srv *server.Server) int {
 	if len(traffic) == 0 {
 		fmt.Fprintln(stderr, "proof: no client sent anything to the server, so there's nothing to check")
 		showDeath(stderr, srv)
 		return 3
 	}
-	r := report.Run{
-		Proof:   version,
-		Server:  "client traffic to " + name,
-		Address: upstream,
-		Started: started,
-		Results: check.Grade(checks.ClientRules(), traffic),
-	}
+	r.Results = check.Grade(checks.ClientRules(), traffic)
 	report.Text(stdout, r, o.verbose)
 	if o.verbose {
 		for _, tr := range traffic {

cmd/proof/proxy_test.go

@@ -50,9 +50,13 @@ func doneServer(t *testing.T) string {
 	return ln.Addr().String()
 }
 
-// proxyRun runs proof proxy with a client that sends the given frames and
-// hangs up, and returns the exit status and output.
-func proxyRun(t *testing.T, frames []string, args ...string) (int, string, string) {
+// command is how the tests run proof proxy and proof serve: until ctx is
+// done, telling listening where clients go.
+type command func(ctx context.Context, args []string, stdout, stderr io.Writer, listening func(addr string)) int
+
+// runWithClient runs a command with a client that sends the given frames
+// and hangs up, and returns the exit status and output.
+func runWithClient(t *testing.T, run command, frames []string, args ...string) (int, string, string) {
 	t.Helper()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -75,11 +79,17 @@ func proxyRun(t *testing.T, frames []string, args ...string) (int, string, strin
 		io.Copy(io.Discard, c)
 	}
 	var stdout, stderr bytes.Buffer
-	args = append([]string{"-address", doneServer(t)}, args...)
-	code := proxyUntil(ctx, args, &stdout, &stderr, client)
+	code := run(ctx, args, &stdout, &stderr, client)
 	return code, stdout.String(), stderr.String()
 }
 
+// proxyRun runs proof proxy in front of a server that answers everything
+// with done.
+func proxyRun(t *testing.T, frames []string, args ...string) (int, string, string) {
+	t.Helper()
+	return runWithClient(t, proxyUntil, frames, append([]string{"-address", doneServer(t)}, args...)...)
+}
+
 func TestProxyExitCodes(t *testing.T) {
 	cases := []struct {
 		name   string
@@ -129,10 +139,12 @@ func TestProxyNeedsAServer(t *testing.T) {
 	}
 }
 
-func TestListShowsClientRules(t *testing.T) {
+func TestListShowsEverything(t *testing.T) {
 	var buf bytes.Buffer
 	list(&buf)
-	if !strings.Contains(buf.String(), "client.need-input") || !strings.Contains(buf.String(), "wire.dict") {
-		t.Errorf("list is missing rules:\n%s", buf.String())
+	for _, want := range []string{"eval.value", "wire.dict", "client.need-input", "split-output"} {
+		if !strings.Contains(buf.String(), want) {
+			t.Errorf("list is missing %s:\n%s", want, buf.String())
+		}
 	}
 }

cmd/proof/serve.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/nrepl/proof/internal/report"
+	"github.com/nrepl/proof/internal/serve"
+)
+
+func serveFlags(out io.Writer, o *clientOptions) *flag.FlagSet {
+	fs := flag.NewFlagSet("serve", flag.ContinueOnError)
+	fs.SetOutput(out)
+	o.register(fs)
+	return fs
+}
+
+// runServe is a server for clients to run their tests against until it's
+// interrupted, and then grades everything the clients sent.
+func runServe(args []string) int {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	return serveUntil(ctx, args, os.Stdout, os.Stderr, nil)
+}
+
+// serveUntil does the work of runServe, grading the traffic once ctx is
+// done. listening, if not nil, gets the address clients should use.
+func serveUntil(ctx context.Context, args []string, stdout, stderr io.Writer, listening func(addr string)) int {
+	// The server logs to stderr from goroutines of its own.
+	stderr = &syncWriter{w: stderr}
+	var o clientOptions
+	fs := serveFlags(stderr, &o)
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	srv, err := serve.Listen(o.listen, version, fs.Args())
+	if err != nil {
+		fmt.Fprintln(stderr, "proof:", err)
+		return 2
+	}
+	srv.Logf = logTo(stderr)
+	go srv.Serve()
+	started := time.Now()
+	name := "proof serve"
+	if fs.NArg() > 0 {
+		name += " (" + strings.Join(fs.Args(), ", ") + ")"
+	}
+	fmt.Fprintf(stderr, "Running %s on %s. Connect your client and press Ctrl-C when it's done.\n", name, srv.Addr())
+	if listening != nil {
+		listening(srv.Addr())
+	}
+
+	<-ctx.Done()
+	fmt.Fprintln(stderr)
+	r := report.Run{Proof: version, Server: "client traffic to " + name, Address: srv.Addr(), Started: started}
+	return gradeClients(stdout, stderr, r, srv.Stop(time.Second), o, nil)
+}

cmd/proof/serve_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestServeExitCodes(t *testing.T) {
+	cases := []struct {
+		name   string
+		args   []string
+		frames []string
+		code   int
+		output string
+	}{
+		{"well-behaved client", nil, []string{"d2:id1:14:code7:(+ 1 2)2:op4:evale"}, 0, "client traffic to proof serve"},
+		{"with scenarios", []string{"last-value", "no-err"}, []string{"d2:id1:12:op8:describee"}, 0,
+			"client traffic to proof serve (last-value, no-err)"},
+		{"request without an id", nil, []string{"d2:op8:describee"}, 1, "a request without an id"},
+		{"no client", nil, nil, 3, "nothing to check"},
+		{"unknown scenario", []string{"no-such-scenario"}, nil, 2, `unknown scenario "no-such-scenario"`},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			code, stdout, stderr := runWithClient(t, serveUntil, c.frames, c.args...)
+			if code != c.code {
+				t.Errorf("exit status %d, want %d\nstdout:\n%s\nstderr:\n%s", code, c.code, stdout, stderr)
+			}
+			if !strings.Contains(stdout+stderr, c.output) {
+				t.Errorf("output doesn't mention %q\nstdout:\n%s\nstderr:\n%s", c.output, stdout, stderr)
+			}
+		})
+	}
+}