Add proof proxy for checking clients

CONTRIBUTING.md

@@ -60,9 +60,11 @@ how to add a check. When submitting a pull request:
 - Make sure every new check is backed by a link to the client code that
   depends on the behavior (for failures) or to nREPL or the spec (for
   warnings). See the [grading rule](doc/design.md#how-checks-are-graded)
-  for details.
+  for details. For client rules it's the other way around - failures
+  need a link to the server code that breaks.
 - Add a quirk to the fake server for every new check, and an entry for
-  it in `internal/checks/checks_test.go`.
+  it in `internal/checks/checks_test.go`. Client rules get a quirk in the
+  scripted client in `internal/checks/client_test.go` instead.
 - Make sure new and updated checks pass against nREPL itself
   (`profiles/clojure.toml`).
 - Run `gofmt -l .`, `go vet ./...` and `go test -race ./...` before

README.md

@@ -8,6 +8,8 @@ proof checks whether an nREPL server will actually work with the clients
 people use (CIDER, Calva, Conjure, vim-fireplace, REPLy and so on). It
 talks to the server over a socket like any client would, runs a set of
 checks against it and tells you what's broken and which clients it breaks.
+It can also check the other side of the conversation, i.e. the requests
+an nREPL client sends.
 
 The [nREPL protocol spec](https://spec.nrepl.org) is still a draft and in
 a few places it disagrees with what clients actually do. When that
@@ -70,10 +72,27 @@ server (or that users will lose data). If your server simply does
 something differently from the reference nREPL implementation, you'll
 get a warning instead.
 
+## Checking Clients
+
+If you're working on a client, `proof proxy` can sit between it and a
+server and record everything the two of them say to each other:
+
+```shell
+$ proof proxy -listen 127.0.0.1:7888 profiles/babashka.toml
+```
+
+Connect your client to port 7888, use it for a while (or run its test
+suite against it) and press Ctrl-C when you're done. proof will then
+check the requests your client sent - e.g. that every request has an
+`id`, that `need-input` gets answered with `stdin` in the right session
+and that sessions get closed in the end. Here a failure means that some
+server won't work properly with your client, and the report links to the
+server code in question.
+
 ## Documentation
 
 - [Usage](doc/usage.md) - checking your server, reading the report,
-  running proof in CI and comparing servers
+  running proof in CI, comparing servers and checking your client
 - [Profiles](doc/profiles.md) - all the profile options, the snippets and
   known failures
 - [Design](doc/design.md) - the general approach and how checks are graded
@@ -87,7 +106,8 @@ get a warning instead.
 
 proof is still in its early days. Right now it covers the core of the
 protocol (`describe`, unknown ops, `eval`, sessions, `stdin` and the wire format),
-and `proof list` will show you all the checks.
+along with the requests clients send, and `proof list` will show you all
+the checks.
 
 Here's what's coming next:
 
@@ -96,6 +116,9 @@ Here's what's coming next:
   clients disconnecting in the middle of an evaluation)
 - replaying what real clients send (e.g. when CIDER or Calva connect to a
   server) as client profiles
+- a server that misbehaves on purpose (late output, output split into
+  many messages and so on), so client test suites can check how their
+  client deals with replies
 - publishing the compatibility matrix somewhere nicer than a CI job
   summary
 

bencode/bencode_test.go

@@ -100,10 +100,14 @@ func TestDecodeSyntaxErrors(t *testing.T) {
 }
 
 func TestDecodeTruncated(t *testing.T) {
-	for _, in := range []string{"i42", "4:sp", "l4:spam", "d2:id"} {
-		if _, err := decodeOne(t, in); !errors.Is(err, io.ErrUnexpectedEOF) {
+	for _, in := range []string{"i42", "4:sp", "l4:spam", "d2:id", "d4:code10:(+ 1"} {
+		v, err := decodeOne(t, in)
+		if !errors.Is(err, io.ErrUnexpectedEOF) {
 			t.Errorf("%q: got %v, want io.ErrUnexpectedEOF", in, err)
 		}
+		if string(v.Raw) != in {
+			t.Errorf("%q: raw is %q", in, v.Raw)
+		}
 	}
 }
 

bencode/decode.go

@@ -65,7 +65,9 @@ func NewDecoder(r io.Reader) *Decoder {
 
 // Value is one decoded top-level value.
 type Value struct {
-	Data       any
+	Data any
+	// Raw is every byte the decoder consumed, which after an error is the
+	// value up to the point where it went wrong.
 	Raw        []byte
 	Violations []Violation
 }
@@ -266,13 +268,14 @@ func (d *Decoder) str() (string, error) {
 		}
 	}
 	buf := make([]byte, n)
-	if _, err := io.ReadFull(d.r, buf); err != nil {
+	read, err := io.ReadFull(d.r, buf)
+	d.off += int64(read)
+	d.raw = append(d.raw, buf[:read]...)
+	if err != nil {
 		if errors.Is(err, io.EOF) {
 			err = io.ErrUnexpectedEOF
 		}
 		return "", err
 	}
-	d.off += int64(n)
-	d.raw = append(d.raw, buf...)
 	return string(buf), nil
 }

cmd/proof/main.go

@@ -1,5 +1,5 @@
 // Command proof checks an nREPL server's compatibility with the clients
-// people actually use.
+// people actually use, and what a client sends to a server.
 package main
 
 import (
@@ -25,15 +25,17 @@ const version = "0.1.0-dev"
 const usage = `proof checks an nREPL server's compatibility with existing clients.
 
 Usage:
-  proof run [flags] PROFILE   run the checks against the server a profile describes
-  proof matrix REPORT...      build a Markdown compatibility matrix from JSON reports
-  proof list                  list every check and wire rule
+  proof run [flags] PROFILE       run the checks against the server a profile describes
+  proof proxy [flags] [PROFILE]   check what a client sends to a server, by sitting between them
+  proof matrix REPORT...          build a Markdown compatibility matrix from JSON reports
+  proof list                      list every check and rule
   proof version
 
 The exit status of run is 0 when everything passed (or failed as the
 profile expects), 1 when the server failed checks, 2 when proof couldn't
 start (bad flags or profile, or the server didn't come up), and 3 when some
-checks couldn't run at all.
+checks couldn't run at all. The same goes for proxy, where 1 means the
+client failed rules and 3 means no client sent anything.
 
 Run flags:
 `
@@ -46,6 +48,8 @@ func main() {
 	switch os.Args[1] {
 	case "run":
 		os.Exit(run(os.Args[2:]))
+	case "proxy":
+		os.Exit(runProxy(os.Args[2:]))
 	case "matrix":
 		os.Exit(matrix(os.Args[2:]))
 	case "list":
@@ -63,6 +67,8 @@ func main() {
 func printUsage(w io.Writer) {
 	fmt.Fprint(w, usage)
 	runFlags(w).PrintDefaults()
+	fmt.Fprint(w, "\nProxy flags:\n")
+	proxyFlags(w, &proxyOptions{}).PrintDefaults()
 }
 
 type options struct {
@@ -137,9 +143,7 @@ func run(args []string) int {
 	for _, id := range outcome.Stale {
 		fmt.Printf("%s is listed in expected-failures but didn't fail; remove it from the profile\n", id)
 	}
-	if srv != nil && !srv.Alive() {
-		fmt.Printf("\nThe server exited during the run. Its output:\n%s", srv.Output())
-	}
+	showDeath(os.Stdout, srv)
 	if opts.json != "" {
 		if err := writeJSON(opts.json, r); err != nil {
 			fmt.Fprintln(os.Stderr, "proof:", err)
@@ -181,6 +185,14 @@ func startServer(p *profile.Profile) (*server.Server, error) {
 	return srv, err
 }
 
+// showDeath shows the output of a server that exited while proof was
+// using it, as that explains whatever went wrong afterwards.
+func showDeath(w io.Writer, srv *server.Server) {
+	if srv != nil && !srv.Alive() {
+		fmt.Fprintf(w, "\nThe server exited during the run. Its output:\n%s", srv.Output())
+	}
+}
+
 func writeJSON(path string, r report.Run) error {
 	f, err := os.Create(path)
 	if err != nil {
@@ -198,16 +210,22 @@ type entry struct {
 	severity  check.Severity
 }
 
-// catalog lists every check and wire rule.
+// catalog lists every check and wire rule, i.e. everything a server
+// profile can expect to fail.
 func catalog() []entry {
 	var all []entry
 	for _, c := range checks.All() {
 		all = append(all, entry{c.ID, c.Title, c.Severity})
 	}
-	for _, r := range checks.WireRules() {
-		all = append(all, entry{r.ID, r.Title, r.Severity})
+	return append(all, ruleEntries(checks.WireRules())...)
+}
+
+func ruleEntries(rules []*check.Rule) []entry {
+	var entries []entry
+	for _, r := range rules {
+		entries = append(entries, entry{r.ID, r.Title, r.Severity})
 	}
-	return all
+	return entries
 }
 
 func allIDs() []string {
@@ -257,7 +275,7 @@ func filter(all []*check.Check, pattern string) ([]*check.Check, error) {
 }
 
 func list(w io.Writer) {
-	for _, e := range catalog() {
+	for _, e := range append(catalog(), ruleEntries(checks.ClientRules())...) {
 		fmt.Fprintf(w, "%-28s %-4s %s\n", e.id, e.severity, e.title)
 	}
 }

cmd/proof/proxy.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/nrepl/proof/internal/check"
+	"github.com/nrepl/proof/internal/checks"
+	"github.com/nrepl/proof/internal/profile"
+	"github.com/nrepl/proof/internal/proxy"
+	"github.com/nrepl/proof/internal/report"
+	"github.com/nrepl/proof/internal/server"
+)
+
+type proxyOptions struct {
+	address string
+	listen  string
+	json    string
+	verbose bool
+}
+
+func proxyFlags(out io.Writer, o *proxyOptions) *flag.FlagSet {
+	fs := flag.NewFlagSet("proxy", flag.ContinueOnError)
+	fs.SetOutput(out)
+	fs.StringVar(&o.address, "address", "", "forward clients to a server already running at `host:port` instead of launching the one the profile describes")
+	fs.StringVar(&o.listen, "listen", "127.0.0.1:0", "accept clients on `host:port` (port 0 picks a free one)")
+	fs.StringVar(&o.json, "json", "", "also write a JSON report to `file`")
+	fs.BoolVar(&o.verbose, "v", false, "show every message the client and the server exchanged")
+	return fs
+}
+
+// runProxy sits between a client and a server until it's interrupted, and
+// then grades everything the client sent.
+func runProxy(args []string) int {
+	// Ctrl-C abandons a server that's still starting, and after that it
+	// means the client is done. The handler stays for the rest of the run,
+	// as the server is in its own process group and has to be stopped by
+	// proof.
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	return proxyUntil(ctx, args, os.Stdout, os.Stderr, nil)
+}
+
+// proxyUntil does the work of runProxy, grading the traffic once ctx is
+// done. listening, if not nil, gets the address clients should use.
+func proxyUntil(ctx context.Context, args []string, stdout, stderr io.Writer, listening func(addr string)) int {
+	// The proxy logs to stderr from goroutines of its own.
+	stderr = &syncWriter{w: stderr}
+	var o proxyOptions
+	fs := proxyFlags(stderr, &o)
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 1 || (fs.NArg() == 0 && o.address == "") {
+		fmt.Fprintln(stderr, "proof proxy: expected a profile or -address")
+		return 2
+	}
+	upstream, name := o.address, "the server"
+	var p *profile.Profile
+	if fs.NArg() == 1 {
+		var err error
+		if p, err = profile.Load(fs.Arg(0)); err != nil {
+			fmt.Fprintln(stderr, "proof:", err)
+			return 2
+		}
+		name = p.Name
+		if upstream == "" {
+			upstream = p.Address
+		}
+	}
+
+	var srv *server.Server
+	if upstream == "" {
+		fmt.Fprintf(stderr, "Starting %s...\n", p.Name)
+		var err error
+		if srv, err = server.Start(ctx, p); err != nil {
+			if ctx.Err() != nil {
+				return 130
+			}
+			fmt.Fprintln(stderr, "proof:", err)
+			return 2
+		}
+		defer srv.Stop()
+		upstream = srv.Addr
+	}
+	px, err := proxy.Listen(o.listen, upstream)
+	if err != nil {
+		fmt.Fprintln(stderr, "proof:", err)
+		return 2
+	}
+	px.Logf = func(format string, args ...any) {
+		fmt.Fprintf(stderr, format+"\n", args...)
+	}
+	go px.Serve()
+	started := time.Now()
+	fmt.Fprintf(stderr, "Forwarding %s to %s. Connect your client to %s and press Ctrl-C when it's done.\n",
+		px.Addr(), upstream, px.Addr())
+	if listening != nil {
+		listening(px.Addr())
+	}
+
+	<-ctx.Done()
+	fmt.Fprintln(stderr)
+	traffic := px.Stop(time.Second)
+	if len(traffic) == 0 {
+		fmt.Fprintln(stderr, "proof: no client sent anything to the server, so there's nothing to check")
+		showDeath(stderr, srv)
+		return 3
+	}
+	r := report.Run{
+		Proof:   version,
+		Server:  "client traffic to " + name,
+		Address: upstream,
+		Started: started,
+		Results: check.Grade(checks.ClientRules(), traffic),
+	}
+	report.Text(stdout, r, o.verbose)
+	if o.verbose {
+		for _, tr := range traffic {
+			fmt.Fprintf(stdout, "\n%s:\n", tr.Label)
+			report.Transcript(stdout, tr.Events, "  ")
+		}
+	}
+	showDeath(stdout, srv)
+	if o.json != "" {
+		if err := writeJSON(o.json, r); err != nil {
+			fmt.Fprintln(stderr, "proof:", err)
+			return 2
+		}
+	}
+	if r.Counts()[check.Failed] > 0 {
+		return 1
+	}
+	return 0
+}
+
+type syncWriter struct {
+	mu sync.Mutex
+	w  io.Writer
+}
+
+func (s *syncWriter) Write(p []byte) (int, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.w.Write(p)
+}