From 5f8503c2eccd32de04428699cc37199a0d94ce1e Mon Sep 17 00:00:00 2001 From: AlphaJack <19962243+AlphaJack@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:09:23 +0100 Subject: [PATCH 1/4] feat: add --http-version to select HTTP/1.1, HTTP/2 or HTTP/3 Test servers can now be reached over HTTP/3 (QUIC, via quic-go). The default, auto, keeps negotiating HTTP/2 or HTTP/1.1 over TCP and moves a server to HTTP/3 once it advertises it through Alt-Svc, falling back to TCP if QUIC then fails. 1.1, 2 and 3 force that version and never fall back. The server list and telemetry stay on TCP. QUIC sockets are bound like the TCP ones, so --source, --interface, --fwmark and -4/-6 apply to HTTP/3 too. Initial packets are 1200 bytes, as quic-go's 1280-byte default does not fit the 1280-byte MTU of WireGuard/Tailscale tunnels (quic-go/quic-go#5573). The protocol used is printed and reported in --json as "protocol". --- README.md | 3 + defs/options.go | 1 + defs/server.go | 5 +- go.mod | 6 +- go.sum | 20 +++- main.go | 6 ++ report/json.go | 2 + speedtest/helper.go | 2 + speedtest/speedtest.go | 12 ++- speedtest/transport.go | 187 ++++++++++++++++++++++++++++++++++++ speedtest/transport_test.go | 115 ++++++++++++++++++++++ 11 files changed, 353 insertions(+), 6 deletions(-) create mode 100644 speedtest/transport.go create mode 100644 speedtest/transport_test.go diff --git a/README.md b/README.md index 478a504..29534e9 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This is a command line interface for LibreSpeed speed test backends, written in - Result sharing (telemetry) *[optional]* - Test with multiple servers in a single run - Use your own server list or telemetry endpoints +- HTTP/1.1, HTTP/2 and HTTP/3 (QUIC), selectable with `--http-version` - Tested with PHP and Go backends [![asciicast](https://asciinema.org/a/J17bUAilWI3qR12JyhfGvPwu2.svg)](https://asciinema.org/a/J17bUAilWI3qR12JyhfGvPwu2) @@ -198,6 +199,8 @@ GLOBAL OPTIONS: --ca-cert value Use the specified CA certificate PEM bundle file instead of the system certificate trust store --skip-cert-verify Skip verifying SSL certificate for HTTPS connections (self-signed certs) (default: false) + --http-version VERSION HTTP VERSION for test servers: 1.1, 2, 3, or auto to + switch to HTTP/3 when advertised and fall back if it fails (default: "auto") --no-pre-allocate Do not pre allocate upload data. Pre allocation is enabled by default to improve upload performance. To support systems with insufficient memory, use this diff --git a/defs/options.go b/defs/options.go index 32a64f5..ec702fa 100644 --- a/defs/options.go +++ b/defs/options.go @@ -45,4 +45,5 @@ const ( OptionTelemetryShare = "telemetry-share" OptionTelemetryExtra = "telemetry-extra" OptionFwmark = "fwmark" + OptionHTTPVersion = "http-version" ) diff --git a/defs/server.go b/defs/server.go index c5349d0..5391b59 100644 --- a/defs/server.go +++ b/defs/server.go @@ -45,6 +45,7 @@ type Server struct { // plain HTTP. Filled by CheckUp (and IsUp, which wraps it), which runs // before every measurement. NegotiatedTLS *TLSInfo `json:"-"` + Protocol string `json:"-"` // e.g. HTTP/3.0, filled by IsUp too } // TLSInfo names the negotiated TLS parameters of a connection. @@ -90,6 +91,8 @@ func (s *Server) CheckUp() error { } defer resp.Body.Close() + s.Protocol = resp.Proto + // Report what the connection actually negotiated. On hardware without AES // acceleration the cipher, not the link, is what bounds the result, and // under TLS 1.3 the server picks it: the client offers a set and has no say @@ -301,7 +304,7 @@ func (s *Server) PingAndJitter(count int) (float64, float64, error) { } for _, addr := range remotes { - output.WriteDebug("Pinging %s over TCP (%s)\n", addr, addressFamily(addr)) + output.WriteDebug("Pinging %s over %s (%s)\n", addr, s.Protocol, addressFamily(addr)) } // discard first result due to handshake overhead diff --git a/go.mod b/go.mod index 347a7fb..4481c50 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/briandowns/spinner v1.23.2 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/prometheus-community/pro-bing v0.9.1 + github.com/quic-go/quic-go v0.63.0 github.com/urfave/cli/v2 v2.27.7 golang.org/x/sys v0.50.0 ) @@ -16,9 +17,12 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/quic-go/qpack v0.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.57.0 // indirect - golang.org/x/sync v0.21.0 // indirect + golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect ) diff --git a/go.sum b/go.sum index ace0742..362f8ef 100644 --- a/go.sum +++ b/go.sum @@ -15,19 +15,35 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/prometheus-community/pro-bing v0.9.1 h1:kpuAr6AU2oRtzGihJSUcetHtjc7ku7h6PxeuW9RVrQw= github.com/prometheus-community/pro-bing v0.9.1/go.mod h1:z79wYTxAOf6FpTng0QdIhZ3j/Nd5l3+gnFSCIT+SiSQ= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A= +github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= diff --git a/main.go b/main.go index 12ea2b1..6a25a96 100644 --- a/main.go +++ b/main.go @@ -225,6 +225,12 @@ func main() { Usage: "firewall mark to set on socket.", Value: 0, }, + &cli.StringFlag{ + Name: defs.OptionHTTPVersion, + Usage: "HTTP `VERSION` for test servers: 1.1, 2, 3, or auto to\n" + + "\tswitch to HTTP/3 when advertised and fall back if it fails", + Value: "auto", + }, }, } diff --git a/report/json.go b/report/json.go index c3b8cc2..4b8d418 100644 --- a/report/json.go +++ b/report/json.go @@ -26,6 +26,8 @@ type JSONReport struct { // it -- so two otherwise identical runs can differ several-fold for a // reason the numbers alone do not show. TLS *TLSReport `json:"tls,omitempty"` + + Protocol string `json:"protocol,omitempty"` // HTTP/1.1, HTTP/2.0 or HTTP/3.0 } // TLSReport names the negotiated TLS parameters a measurement ran over. diff --git a/speedtest/helper.go b/speedtest/helper.go index b980261..5684d72 100644 --- a/speedtest/helper.go +++ b/speedtest/helper.go @@ -64,6 +64,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel } if upErr := currentServer.CheckUp(); upErr == nil { + output.WriteUI("Protocol: %s\n", currentServer.Protocol) output.WriteDebug("Fetching IP info\n") ispInfo, err := currentServer.GetIPInfo(c.String(defs.OptionDistance)) if err != nil { @@ -217,6 +218,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel rep.Server.Name = currentServer.Name rep.Server.URL = u.String() + rep.Protocol = currentServer.Protocol if currentServer.NegotiatedTLS != nil { rep.TLS = &report.TLSReport{ Version: currentServer.NegotiatedTLS.Version, diff --git a/speedtest/speedtest.go b/speedtest/speedtest.go index e06c5ba..9229d3b 100644 --- a/speedtest/speedtest.go +++ b/speedtest/speedtest.go @@ -252,7 +252,11 @@ func SpeedTest(c *cli.Context) error { // set default HTTP client's Transport to the one that binds the source address // this is modified from http.DefaultTransport transport.DialContext = dialContext - http.DefaultClient.Transport = transport + pt, err := newProtoTransport(c.String(defs.OptionHTTPVersion), transport, network, dialer) + if err != nil { + return err + } + http.DefaultClient.Transport = pt // no scheme is forced by default // force https if --secure is given @@ -266,7 +270,6 @@ func SpeedTest(c *cli.Context) error { // load server list var servers []defs.Server - var err error if str := c.String(defs.OptionLocalJSON); str != "" { switch str { case "-": @@ -297,6 +300,11 @@ func SpeedTest(c *cli.Context) error { output.WriteError("Error when fetching server list: %s\n", err) return err } + for _, s := range servers { + if u, err := s.GetURL(); err == nil { + pt.addServer(u.Host) + } + } // if --list is given, list all the servers fetched and exit if c.Bool(defs.OptionList) { diff --git a/speedtest/transport.go b/speedtest/transport.go new file mode 100644 index 0000000..c9700a7 --- /dev/null +++ b/speedtest/transport.go @@ -0,0 +1,187 @@ +package speedtest + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" + + "github.com/librespeed/speedtest-cli/output" +) + +// protoTransport sends requests to the test servers over the HTTP version +// given with --http-version; other requests (server list, telemetry) use base. +// In auto mode a server moves to HTTP/3 once it advertises it via Alt-Svc, +// and back to TCP if QUIC then fails. A forced version never falls back. +type protoTransport struct { + version string + base http.RoundTripper + tcp *http.Transport + h3 *http3.Transport + + mu sync.Mutex + hosts map[string]string // test servers: "", "h3" or "tcp" in auto mode +} + +func newProtoTransport(version string, tcp *http.Transport, network string, dialer *net.Dialer) (*protoTransport, error) { + t := &protoTransport{version: version, tcp: tcp, hosts: map[string]string{}} + switch version { + case "auto", "3": + case "1.1": + t.tcp.Protocols = new(http.Protocols) + t.tcp.Protocols.SetHTTP1(true) + case "2": + t.tcp.Protocols = new(http.Protocols) + t.tcp.Protocols.SetHTTP2(true) + t.tcp.Protocols.SetUnencryptedHTTP2(true) + default: + return nil, fmt.Errorf("unsupported HTTP version %q, use one of auto, 1.1, 2, 3", version) + } + // clone only once Protocols is set: Clone fixes the source's protocols + base := tcp.Clone() + base.Protocols = nil + t.base = base + t.h3 = &http3.Transport{ + TLSClientConfig: tcp.TLSClientConfig.Clone(), + // quic-go's default of 1280 does not fit the 1280-byte MTU of + // WireGuard/Tailscale tunnels (quic-go/quic-go#5573) + QUICConfig: &quic.Config{InitialPacketSize: 1200}, + Dial: func(ctx context.Context, addr string, tlsConf *tls.Config, quicConf *quic.Config) (*quic.Conn, error) { + return dialQUIC(ctx, network, dialer, addr, tlsConf, quicConf) + }, + } + return t, nil +} + +// addServer marks a host as a test server, to be reached over --http-version. +func (t *protoTransport) addServer(host string) { + t.mu.Lock() + defer t.mu.Unlock() + if _, ok := t.hosts[host]; !ok { + t.hosts[host] = "" + } +} + +func (t *protoTransport) RoundTrip(req *http.Request) (*http.Response, error) { + t.mu.Lock() + state, isServer := t.hosts[req.URL.Host] + t.mu.Unlock() + switch { + case !isServer: + return t.base.RoundTrip(req) + case t.version == "3": + if req.URL.Scheme != "https" { + return nil, errors.New("HTTP/3 requires https, try --secure") + } + resp, err := t.h3.RoundTrip(req) + if err != nil && req.Context().Err() == nil { + output.WriteError("HTTP/3 to %s failed: %s\n", output.Sanitize(req.URL.Host), err) + } + return resp, err + case t.version != "auto" || req.URL.Scheme != "https": + return t.tcp.RoundTrip(req) + } + + if state == "h3" { + resp, err := t.h3.RoundTrip(req) + if err == nil || req.Context().Err() != nil { + return resp, err + } + output.WriteUI("HTTP/3 to %s failed, falling back to TCP: %s\n", output.Sanitize(req.URL.Host), err) + t.setState(req.URL.Host, "tcp") + if !replayable(req) { + return nil, err + } + return t.tcp.RoundTrip(req) + } + + resp, err := t.tcp.RoundTrip(req) + if err != nil || state != "" || !advertisesH3(resp, req.URL) { + return resp, err + } + output.WriteDebug("%s advertises HTTP/3, switching to it\n", output.Sanitize(req.URL.Host)) + t.setState(req.URL.Host, "h3") + // repeat the request over HTTP/3 when possible, so what IsUp reports is + // what the test then runs over + if !replayable(req) { + return resp, nil + } + resp.Body.Close() + return t.RoundTrip(req) +} + +func (t *protoTransport) setState(host, state string) { + t.mu.Lock() + defer t.mu.Unlock() + t.hosts[host] = state +} + +// advertisesH3 reports whether resp offers HTTP/3 on the same host and port. +func advertisesH3(resp *http.Response, u *url.URL) bool { + port := u.Port() + if port == "" { + port = "443" + } + for _, v := range resp.Header.Values("Alt-Svc") { + if strings.Contains(v, `h3=":`+port+`"`) { + return true + } + } + return false +} + +func replayable(req *http.Request) bool { + return req.Method == http.MethodGet && (req.Body == nil || req.Body == http.NoBody) +} + +// dialQUIC opens a QUIC connection on its own UDP socket, bound like the TCP +// dialer's sockets (source address, interface, fwmark, IP family). +func dialQUIC(ctx context.Context, network string, dialer *net.Dialer, addr string, tlsConf *tls.Config, quicConf *quic.Config) (*quic.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + ips, err := net.DefaultResolver.LookupIP(ctx, network, host) + if err != nil { + return nil, err + } + laddr := "" + if src, ok := dialer.LocalAddr.(*net.TCPAddr); ok { + laddr = net.JoinHostPort(src.IP.String(), "0") + } + var errs []error + for _, ip := range ips { + udp := "udp6" + if ip.To4() != nil { + udp = "udp4" + } + raddr, err := net.ResolveUDPAddr(udp, net.JoinHostPort(ip.String(), port)) + if err != nil { + return nil, err + } + pc, err := (&net.ListenConfig{Control: dialer.Control}).ListenPacket(ctx, udp, laddr) + if err != nil { + return nil, err + } + tr := &quic.Transport{Conn: pc} + conn, err := tr.DialEarly(ctx, raddr, tlsConf, quicConf) + if err == nil { + go func() { <-conn.Context().Done(); tr.Close() }() + return conn, nil + } + tr.Close() + errs = append(errs, fmt.Errorf("%s: %w", ip, err)) + if ctx.Err() != nil { + break + } + } + return nil, errors.Join(errs...) +} diff --git a/speedtest/transport_test.go b/speedtest/transport_test.go new file mode 100644 index 0000000..48302d4 --- /dev/null +++ b/speedtest/transport_test.go @@ -0,0 +1,115 @@ +package speedtest + +import ( + "crypto/tls" + "crypto/x509" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/quic-go/quic-go/http3" +) + +// testServer serves TCP and, if h3, QUIC on the same port, replying with the +// protocol it was reached over and advertising HTTP/3 if asked to. +func testServer(t *testing.T, h3, advertise bool) *httptest.Server { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, port, _ := net.SplitHostPort(r.Host); advertise { + w.Header().Set("Alt-Svc", `h3=":`+port+`"; ma=60`) + } + io.Copy(io.Discard, r.Body) + io.WriteString(w, r.Proto) + }) + srv := httptest.NewUnstartedServer(handler) + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + if h3 { + udp, err := net.ListenPacket("udp", srv.Listener.Addr().String()) + if err != nil { + t.Skip(err) + } + s := &http3.Server{Handler: handler, TLSConfig: http3.ConfigureTLSConfig(srv.TLS.Clone())} + go s.Serve(udp) + t.Cleanup(func() { s.Close() }) + } + return srv +} + +func newTestClient(t *testing.T, version string, srv *httptest.Server) *http.Client { + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + base := http.DefaultTransport.(*http.Transport).Clone() + base.TLSClientConfig = &tls.Config{RootCAs: pool} + pt, err := newProtoTransport(version, base, "ip", &net.Dialer{}) + if err != nil { + t.Fatal(err) + } + u, _ := url.Parse(srv.URL) + pt.addServer(u.Host) + return &http.Client{Transport: pt, Timeout: 10 * time.Second} +} + +func protoOf(t *testing.T, c *http.Client, method, url string) string { + t.Helper() + var body io.Reader + if method == http.MethodPost { + body = strings.NewReader("x") + } + req, _ := http.NewRequest(method, url, body) + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if string(b) != resp.Proto { + t.Fatalf("client saw %s, server saw %s", resp.Proto, b) + } + return resp.Proto +} + +func TestHTTPVersions(t *testing.T) { + srv := testServer(t, true, false) + for version, want := range map[string]string{"1.1": "HTTP/1.1", "2": "HTTP/2.0", "3": "HTTP/3.0", "auto": "HTTP/2.0"} { + t.Run(version, func(t *testing.T) { + if got := protoOf(t, newTestClient(t, version, srv), http.MethodGet, srv.URL); got != want { + t.Errorf("used %s, want %s", got, want) + } + }) + } + if _, err := newProtoTransport("4", &http.Transport{}, "ip", &net.Dialer{}); err == nil { + t.Error("HTTP version 4 accepted") + } +} + +func TestAutoUpgradesOnAltSvc(t *testing.T) { + srv := testServer(t, true, true) + c := newTestClient(t, "auto", srv) + if got := protoOf(t, c, http.MethodGet, srv.URL); got != "HTTP/3.0" { + t.Fatalf("first request used %s, want HTTP/3.0", got) + } + if got := protoOf(t, c, http.MethodPost, srv.URL); got != "HTTP/3.0" { + t.Fatalf("upload used %s, want HTTP/3.0", got) + } +} + +func TestAutoFallsBackWhenQUICFails(t *testing.T) { + srv := testServer(t, false, true) + c := newTestClient(t, "auto", srv) + for range 2 { + if got := protoOf(t, c, http.MethodGet, srv.URL); got != "HTTP/2.0" { + t.Fatalf("used %s, want fallback to HTTP/2.0", got) + } + } + c = newTestClient(t, "3", srv) + c.Timeout = 2 * time.Second + if _, err := c.Get(srv.URL); err == nil { + t.Fatal("forced HTTP/3 succeeded without a QUIC server") + } +} From 9821b42d00b073dcb5828b6c6452739f1a9b5113 Mon Sep 17 00:00:00 2001 From: AlphaJack <19962243+AlphaJack@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:09:24 +0100 Subject: [PATCH 2/4] fix: address review of --http-version - Keep HTTP/3 off behind an HTTP(S) proxy: auto stays on TCP, forced 3 errors - Skip addresses of the other IP family when --source is set - Cap the QUIC handshake in auto at a third of --timeout so the TCP retry fits - Parse Alt-Svc per RFC 7838, honouring ma=0 and clear - Sanitize error text from the server before printing it - Report the protocol and TLS the measurements ran over - Send telemetry over a TCP-only client - Show the "HTTP/3 requires https" hint without --debug - Bump quic-go to v0.65.0, which fixes slow uploads on reordering links --- README.md | 4 +- go.mod | 12 +- go.sum | 24 ++-- main.go | 4 +- speedtest/helper.go | 31 ++++- speedtest/speedtest.go | 5 +- speedtest/transport.go | 237 +++++++++++++++++++++++++++++++----- speedtest/transport_test.go | 168 ++++++++++++++++++++++++- 8 files changed, 432 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 29534e9..0abd605 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,9 @@ GLOBAL OPTIONS: of the system certificate trust store --skip-cert-verify Skip verifying SSL certificate for HTTPS connections (self-signed certs) (default: false) --http-version VERSION HTTP VERSION for test servers: 1.1, 2, 3, or auto to - switch to HTTP/3 when advertised and fall back if it fails (default: "auto") + switch to HTTP/3 when advertised and fall back if it fails. + HTTP/3 needs https (see --secure) and is not used through + an HTTP(S)_PROXY (default: "auto") --no-pre-allocate Do not pre allocate upload data. Pre allocation is enabled by default to improve upload performance. To support systems with insufficient memory, use this diff --git a/go.mod b/go.mod index 4481c50..7c622ca 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/briandowns/spinner v1.23.2 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/prometheus-community/pro-bing v0.9.1 - github.com/quic-go/quic-go v0.63.0 + github.com/quic-go/quic-go v0.65.0 github.com/urfave/cli/v2 v2.27.7 golang.org/x/sys v0.50.0 ) @@ -20,9 +20,9 @@ require ( github.com/quic-go/qpack v0.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/net v0.57.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/crypto v0.59.0 // indirect + golang.org/x/net v0.62.0 // indirect + golang.org/x/sync v0.25.0 // indirect + golang.org/x/term v0.48.0 // indirect + golang.org/x/text v0.44.0 // indirect ) diff --git a/go.sum b/go.sum index 362f8ef..c90d5b7 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4 github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A= -github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= +github.com/quic-go/quic-go v0.65.0 h1:BJQSjdTvLPLmWtaCupUphRbgxgpZhdO/GMpr9Eoe3B4= +github.com/quic-go/quic-go v0.65.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= @@ -33,17 +33,17 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c= +golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= +golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk= +golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= diff --git a/main.go b/main.go index 6a25a96..84d9bb1 100644 --- a/main.go +++ b/main.go @@ -228,7 +228,9 @@ func main() { &cli.StringFlag{ Name: defs.OptionHTTPVersion, Usage: "HTTP `VERSION` for test servers: 1.1, 2, 3, or auto to\n" + - "\tswitch to HTTP/3 when advertised and fall back if it fails", + "\tswitch to HTTP/3 when advertised and fall back if it fails.\n" + + "\tHTTP/3 needs https (see --secure) and is not used through\n" + + "\tan HTTP(S)_PROXY", Value: "auto", }, }, diff --git a/speedtest/helper.go b/speedtest/helper.go index 5684d72..acbc86f 100644 --- a/speedtest/helper.go +++ b/speedtest/helper.go @@ -2,6 +2,7 @@ package speedtest import ( "bytes" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -27,6 +28,14 @@ const ( pingCount = 10 ) +// activeTransport routes the test servers' requests; nil until SpeedTest +// sets it up. telemetryClient sends results over TCP only, even when the +// telemetry server is also a test server reached over HTTP/3. +var ( + activeTransport *protoTransport + telemetryClient = http.DefaultClient +) + // doSpeedTest is where the actual speed test happens func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.TelemetryServer, network string, silent bool, noICMP bool) error { // With nothing to test the report would be empty, like when every server @@ -109,6 +118,12 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel output.WriteUI("Ping: %.2f ms\tJitter: %.2f ms\n", p, jitter) } + // report what the measurements ran over, which in auto mode can + // differ from what the probe above saw + if activeTransport != nil { + activeTransport.resetObserved(u.Host) + } + // get download value var downloadValue float64 var bytesRead uint64 @@ -153,6 +168,20 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel output.WriteDebug("Upload test finished in %s: %s, %d byte(s) sent\n", time.Since(uploadStart).Round(time.Millisecond), humanizeRate(upload, c), bw) } + if activeTransport != nil { + if proto, state, ok := activeTransport.observedOn(u.Host); ok && proto != currentServer.Protocol { + output.WriteUI("Protocol used by the test: %s\n", proto) + currentServer.Protocol = proto + currentServer.NegotiatedTLS = nil + if state != nil { + currentServer.NegotiatedTLS = &defs.TLSInfo{ + Version: tls.VersionName(state.Version), + Cipher: tls.CipherSuiteName(state.CipherSuite), + } + } + } + } + // print result if --simple is given if c.Bool(defs.OptionSimple) { if c.Bool(defs.OptionBytes) { @@ -378,7 +407,7 @@ func sendTelemetry(telemetryServer defs.TelemetryServer, ispInfo *defs.GetIPResu req.Header.Set("Content-Type", wr.FormDataContentType()) req.Header.Set("User-Agent", defs.UserAgent) - resp, err := http.DefaultClient.Do(req) + resp, err := telemetryClient.Do(req) if err != nil { output.WriteDebug("Error when making HTTP request: %s\n", err) return "", err diff --git a/speedtest/speedtest.go b/speedtest/speedtest.go index 9229d3b..ac7f129 100644 --- a/speedtest/speedtest.go +++ b/speedtest/speedtest.go @@ -252,11 +252,14 @@ func SpeedTest(c *cli.Context) error { // set default HTTP client's Transport to the one that binds the source address // this is modified from http.DefaultTransport transport.DialContext = dialContext - pt, err := newProtoTransport(c.String(defs.OptionHTTPVersion), transport, network, dialer) + pt, err := newProtoTransport(c.String(defs.OptionHTTPVersion), transport, network, dialer, http.DefaultClient.Timeout) if err != nil { return err } http.DefaultClient.Transport = pt + activeTransport = pt + // telemetry stays on TCP, even when it shares a host with a test server + telemetryClient = &http.Client{Transport: pt.base, Timeout: http.DefaultClient.Timeout} // no scheme is forced by default // force https if --secure is given diff --git a/speedtest/transport.go b/speedtest/transport.go index c9700a7..2cd0903 100644 --- a/speedtest/transport.go +++ b/speedtest/transport.go @@ -8,8 +8,11 @@ import ( "net" "net/http" "net/url" + "slices" + "strconv" "strings" "sync" + "time" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" @@ -17,22 +20,36 @@ import ( "github.com/librespeed/speedtest-cli/output" ) +// maxH3Handshake caps how long auto mode waits for a QUIC handshake before +// it falls back to TCP. Networks that drop UDP give no error, only silence. +const maxH3Handshake = 3 * time.Second + // protoTransport sends requests to the test servers over the HTTP version // given with --http-version; other requests (server list, telemetry) use base. // In auto mode a server moves to HTTP/3 once it advertises it via Alt-Svc, // and back to TCP if QUIC then fails. A forced version never falls back. +// HTTP/3 is never used through a proxy: QUIC cannot go through one. type protoTransport struct { version string - base http.RoundTripper + base *http.Transport tcp *http.Transport h3 *http3.Transport - mu sync.Mutex - hosts map[string]string // test servers: "", "h3" or "tcp" in auto mode + mu sync.Mutex + hosts map[string]string // test servers: "", "h3" or "tcp" in auto mode + observed map[string]*observation +} + +// observation is what the responses from one test server actually ran over. +type observation struct { + protocols []string // in order of first use + tls *tls.ConnectionState } -func newProtoTransport(version string, tcp *http.Transport, network string, dialer *net.Dialer) (*protoTransport, error) { - t := &protoTransport{version: version, tcp: tcp, hosts: map[string]string{}} +// newProtoTransport wraps tcp. timeout is the client's timeout, which bounds +// the QUIC handshake in auto mode so a TCP retry still fits in it. +func newProtoTransport(version string, tcp *http.Transport, network string, dialer *net.Dialer, timeout time.Duration) (*protoTransport, error) { + t := &protoTransport{version: version, tcp: tcp, hosts: map[string]string{}, observed: map[string]*observation{}} switch version { case "auto", "3": case "1.1": @@ -46,14 +63,19 @@ func newProtoTransport(version string, tcp *http.Transport, network string, dial return nil, fmt.Errorf("unsupported HTTP version %q, use one of auto, 1.1, 2, 3", version) } // clone only once Protocols is set: Clone fixes the source's protocols - base := tcp.Clone() - base.Protocols = nil - t.base = base - t.h3 = &http3.Transport{ - TLSClientConfig: tcp.TLSClientConfig.Clone(), + t.base = tcp.Clone() + t.base.Protocols = nil + quicConf := &quic.Config{ // quic-go's default of 1280 does not fit the 1280-byte MTU of // WireGuard/Tailscale tunnels (quic-go/quic-go#5573) - QUICConfig: &quic.Config{InitialPacketSize: 1200}, + InitialPacketSize: 1200, + } + if version == "auto" { + quicConf.HandshakeIdleTimeout = handshakeBudget(timeout) + } + t.h3 = &http3.Transport{ + TLSClientConfig: tcp.TLSClientConfig.Clone(), + QUICConfig: quicConf, Dial: func(ctx context.Context, addr string, tlsConf *tls.Config, quicConf *quic.Config) (*quic.Conn, error) { return dialQUIC(ctx, network, dialer, addr, tlsConf, quicConf) }, @@ -61,6 +83,14 @@ func newProtoTransport(version string, tcp *http.Transport, network string, dial return t, nil } +// handshakeBudget leaves at least two thirds of the client timeout for TCP. +func handshakeBudget(timeout time.Duration) time.Duration { + if timeout > 0 && timeout/3 < maxH3Handshake { + return timeout / 3 + } + return maxH3Handshake +} + // addServer marks a host as a test server, to be reached over --http-version. func (t *protoTransport) addServer(host string) { t.mu.Lock() @@ -74,28 +104,50 @@ func (t *protoTransport) RoundTrip(req *http.Request) (*http.Response, error) { t.mu.Lock() state, isServer := t.hosts[req.URL.Host] t.mu.Unlock() - switch { - case !isServer: + if !isServer { return t.base.RoundTrip(req) + } + resp, err := t.roundTrip(req, state) + if err == nil { + t.observe(req.URL.Host, resp) + } + return resp, err +} + +func (t *protoTransport) roundTrip(req *http.Request, state string) (*http.Response, error) { + switch { case t.version == "3": if req.URL.Scheme != "https" { - return nil, errors.New("HTTP/3 requires https, try --secure") + err := errors.New("HTTP/3 requires https, try --secure") + output.WriteError("%s\n", h3Failure(req.URL.Host, err)) + return nil, err + } + if proxy, err := t.proxyFor(req); proxy != "" || err != nil { + err = proxyError(proxy, err) + output.WriteError("%s\n", h3Failure(req.URL.Host, err)) + return nil, err } resp, err := t.h3.RoundTrip(req) if err != nil && req.Context().Err() == nil { - output.WriteError("HTTP/3 to %s failed: %s\n", output.Sanitize(req.URL.Host), err) + output.WriteError("%s\n", h3Failure(req.URL.Host, err)) } return resp, err case t.version != "auto" || req.URL.Scheme != "https": return t.tcp.RoundTrip(req) } + // auto: a proxy keeps every request on TCP, through the proxy + if proxy, err := t.proxyFor(req); proxy != "" || err != nil { + return t.tcp.RoundTrip(req) + } + if state == "h3" { resp, err := t.h3.RoundTrip(req) + // no time left for TCP once the caller's context is done if err == nil || req.Context().Err() != nil { return resp, err } - output.WriteUI("HTTP/3 to %s failed, falling back to TCP: %s\n", output.Sanitize(req.URL.Host), err) + output.WriteUI("%s, falling back to TCP\n", h3Failure(req.URL.Host, err)) t.setState(req.URL.Host, "tcp") if !replayable(req) { return nil, err @@ -109,13 +161,40 @@ func (t *protoTransport) RoundTrip(req *http.Request) (*http.Response, error) { } output.WriteDebug("%s advertises HTTP/3, switching to it\n", output.Sanitize(req.URL.Host)) t.setState(req.URL.Host, "h3") - // repeat the request over HTTP/3 when possible, so what IsUp reports is - // what the test then runs over + // repeat the request over HTTP/3 when possible, so what CheckUp reports + // is what the test then runs over if !replayable(req) { return resp, nil } resp.Body.Close() - return t.RoundTrip(req) + return t.roundTrip(req, "h3") +} + +// proxyFor returns the proxy the TCP transport would send req through. +func (t *protoTransport) proxyFor(req *http.Request) (string, error) { + if t.tcp.Proxy == nil { + return "", nil + } + u, err := t.tcp.Proxy(req) + if err != nil || u == nil { + return "", err + } + return u.Redacted(), nil +} + +func proxyError(proxy string, err error) error { + if err != nil { + return fmt.Errorf("cannot check the proxy settings: %w", err) + } + return fmt.Errorf("HTTP/3 cannot go through the proxy %s, use --http-version 2 or unset HTTPS_PROXY", proxy) +} + +// h3Failure describes a failed HTTP/3 request. err can carry text from the +// server, such as certificate names, so it is sanitized like the host. +func h3Failure(host string, err error) string { + // errors.Join puts each address's error on its own line + msg := strings.ReplaceAll(err.Error(), "\n", "; ") + return fmt.Sprintf("HTTP/3 to %s failed: %s", output.Sanitize(host), output.Sanitize(msg)) } func (t *protoTransport) setState(host, state string) { @@ -124,20 +203,100 @@ func (t *protoTransport) setState(host, state string) { t.hosts[host] = state } +func (t *protoTransport) observe(host string, resp *http.Response) { + t.mu.Lock() + defer t.mu.Unlock() + o := t.observed[host] + if o == nil { + o = &observation{} + t.observed[host] = o + } + if !slices.Contains(o.protocols, resp.Proto) { + o.protocols = append(o.protocols, resp.Proto) + } + o.tls = resp.TLS +} + +// resetObserved forgets what host's earlier responses ran over. +func (t *protoTransport) resetObserved(host string) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.observed, host) +} + +// observedOn returns the protocols host's responses ran over since the last +// resetObserved, joined with "+" if auto mode switched midway, and the TLS +// state of the last one. ok is false if there was no response. +func (t *protoTransport) observedOn(host string) (protocol string, state *tls.ConnectionState, ok bool) { + t.mu.Lock() + defer t.mu.Unlock() + o := t.observed[host] + if o == nil { + return "", nil, false + } + return strings.Join(o.protocols, "+"), o.tls, true +} + // advertisesH3 reports whether resp offers HTTP/3 on the same host and port. func advertisesH3(resp *http.Response, u *url.URL) bool { port := u.Port() if port == "" { port = "443" } - for _, v := range resp.Header.Values("Alt-Svc") { - if strings.Contains(v, `h3=":`+port+`"`) { + for _, alt := range parseAltSvc(resp.Header.Values("Alt-Svc")) { + if alt.protocol == "h3" && (alt.host == "" || strings.EqualFold(alt.host, u.Hostname())) && alt.port == port { return true } } return false } +type altSvc struct { + protocol, host, port string +} + +// parseAltSvc parses Alt-Svc header values (RFC 7838). It returns no +// alternatives if any value is "clear", and skips the ones withdrawn with +// ma=0 and the malformed ones. +func parseAltSvc(values []string) []altSvc { + var alts []altSvc + for _, v := range values { + for entry := range strings.SplitSeq(v, ",") { + entry = strings.TrimSpace(entry) + if entry == "clear" { + return nil + } + params := strings.Split(entry, ";") + protocol, authority, ok := strings.Cut(params[0], "=") + if !ok { + continue + } + authority, err := strconv.Unquote(strings.TrimSpace(authority)) + if err != nil { + continue + } + host, port, err := net.SplitHostPort(authority) + if err != nil { + continue + } + if p, err := strconv.ParseUint(port, 10, 16); err != nil || p == 0 { + continue + } + withdrawn := false + for _, param := range params[1:] { + name, value, _ := strings.Cut(strings.TrimSpace(param), "=") + if strings.EqualFold(strings.TrimSpace(name), "ma") && strings.Trim(strings.TrimSpace(value), `"`) == "0" { + withdrawn = true + } + } + if !withdrawn { + alts = append(alts, altSvc{strings.TrimSpace(protocol), host, port}) + } + } + } + return alts +} + func replayable(req *http.Request) bool { return req.Method == http.MethodGet && (req.Body == nil || req.Body == http.NoBody) } @@ -153,23 +312,36 @@ func dialQUIC(ctx context.Context, network string, dialer *net.Dialer, addr stri if err != nil { return nil, err } - laddr := "" - if src, ok := dialer.LocalAddr.(*net.TCPAddr); ok { - laddr = net.JoinHostPort(src.IP.String(), "0") - } + return dialQUICAddrs(ctx, ips, port, dialer, tlsConf, quicConf) +} + +// dialQUICAddrs tries each address in turn. With a source address it skips +// the addresses of the other family, which that source cannot reach. +func dialQUICAddrs(ctx context.Context, ips []net.IP, port string, dialer *net.Dialer, tlsConf *tls.Config, quicConf *quic.Config) (*quic.Conn, error) { + src, _ := dialer.LocalAddr.(*net.TCPAddr) var errs []error for _, ip := range ips { udp := "udp6" if ip.To4() != nil { udp = "udp4" } + var laddr *net.UDPAddr + if src != nil { + if (src.IP.To4() != nil) != (ip.To4() != nil) { + errs = append(errs, fmt.Errorf("%s: not reachable from source address %s", ip, src.IP)) + continue + } + laddr = &net.UDPAddr{IP: src.IP, Zone: src.Zone} + } raddr, err := net.ResolveUDPAddr(udp, net.JoinHostPort(ip.String(), port)) if err != nil { - return nil, err + errs = append(errs, err) + continue } - pc, err := (&net.ListenConfig{Control: dialer.Control}).ListenPacket(ctx, udp, laddr) + pc, err := (&net.ListenConfig{Control: dialer.Control}).ListenPacket(ctx, udp, udpAddrString(laddr)) if err != nil { - return nil, err + errs = append(errs, fmt.Errorf("%s: %w", ip, err)) + continue } tr := &quic.Transport{Conn: pc} conn, err := tr.DialEarly(ctx, raddr, tlsConf, quicConf) @@ -185,3 +357,12 @@ func dialQUIC(ctx context.Context, network string, dialer *net.Dialer, addr stri } return nil, errors.Join(errs...) } + +// udpAddrString formats a local address for ListenPacket, keeping the IPv6 +// zone; nil means any address and port. +func udpAddrString(a *net.UDPAddr) string { + if a == nil { + return "" + } + return a.String() +} diff --git a/speedtest/transport_test.go b/speedtest/transport_test.go index 48302d4..c455142 100644 --- a/speedtest/transport_test.go +++ b/speedtest/transport_test.go @@ -1,14 +1,17 @@ package speedtest import ( + "context" "crypto/tls" "crypto/x509" + "errors" "io" "net" "net/http" "net/http/httptest" "net/url" "strings" + "sync/atomic" "testing" "time" @@ -42,17 +45,26 @@ func testServer(t *testing.T, h3, advertise bool) *httptest.Server { } func newTestClient(t *testing.T, version string, srv *httptest.Server) *http.Client { + return newTestClientWith(t, version, srv, nil, 10*time.Second) +} + +// newTestClientWith also sets the proxy the TCP transport would use. +func newTestClientWith(t *testing.T, version string, srv *httptest.Server, proxy *url.URL, timeout time.Duration) *http.Client { pool := x509.NewCertPool() pool.AddCert(srv.Certificate()) base := http.DefaultTransport.(*http.Transport).Clone() base.TLSClientConfig = &tls.Config{RootCAs: pool} - pt, err := newProtoTransport(version, base, "ip", &net.Dialer{}) + base.Proxy = nil + if proxy != nil { + base.Proxy = http.ProxyURL(proxy) + } + pt, err := newProtoTransport(version, base, "ip", &net.Dialer{}, timeout) if err != nil { t.Fatal(err) } u, _ := url.Parse(srv.URL) pt.addServer(u.Host) - return &http.Client{Transport: pt, Timeout: 10 * time.Second} + return &http.Client{Transport: pt, Timeout: timeout} } func protoOf(t *testing.T, c *http.Client, method, url string) string { @@ -83,11 +95,98 @@ func TestHTTPVersions(t *testing.T) { } }) } - if _, err := newProtoTransport("4", &http.Transport{}, "ip", &net.Dialer{}); err == nil { + if _, err := newProtoTransport("4", &http.Transport{}, "ip", &net.Dialer{}, 0); err == nil { t.Error("HTTP version 4 accepted") } } +// A proxy keeps HTTP/3 off: auto stays on TCP, through the proxy, and a +// forced HTTP/3 fails instead of going around it. The proxy is set on the +// transport rather than with HTTPS_PROXY: http.ProxyFromEnvironment reads the +// environment only once per process, and never proxies localhost. +func TestProxyKeepsHTTP3Off(t *testing.T) { + srv := testServer(t, true, true) + var proxied atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + http.Error(w, "CONNECT only", http.StatusMethodNotAllowed) + return + } + proxied.Add(1) + dst, err := net.Dial("tcp", r.Host) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusOK) + src, _, err := http.NewResponseController(w).Hijack() + if err != nil { + dst.Close() + return + } + go func() { io.Copy(dst, src); dst.Close() }() + io.Copy(src, dst) + src.Close() + })) + t.Cleanup(proxy.Close) + proxyURL, _ := url.Parse(proxy.URL) + + c := newTestClientWith(t, "auto", srv, proxyURL, 10*time.Second) + for range 2 { + if got := protoOf(t, c, http.MethodGet, srv.URL); got != "HTTP/2.0" { + t.Fatalf("auto with a proxy used %s, want HTTP/2.0", got) + } + } + if proxied.Load() == 0 { + t.Error("auto did not go through the proxy") + } + + c = newTestClientWith(t, "3", srv, proxyURL, 10*time.Second) + _, err := c.Get(srv.URL) + if err == nil || !strings.Contains(err.Error(), "proxy") { + t.Fatalf("forced HTTP/3 with a proxy: got %v, want a proxy error", err) + } +} + +// A dual-stack host resolved IPv6 first, with an IPv4 source address, must +// still be reached over IPv4. +func TestDialQUICSkipsOtherFamily(t *testing.T) { + srv := testServer(t, true, false) + _, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + tlsConf := &tls.Config{RootCAs: pool, ServerName: "example.com", NextProtos: []string{http3.NextProtoH3}} + dialer := &net.Dialer{LocalAddr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1")}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := dialQUICAddrs(ctx, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1")}, port, dialer, tlsConf, nil) + if err != nil { + t.Fatal(err) + } + conn.CloseWithError(0, "") +} + +func TestH3FailureIsSanitized(t *testing.T) { + msg := h3Failure("host\x1b[2J", errors.New("x509: certificate is valid for \x1b[2Jevil\n")) + if strings.ContainsAny(msg, "\x1b\n") { + t.Errorf("control characters kept: %q", msg) + } +} + +func TestHTTP3NeedsHTTPS(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(srv.Close) + pt, err := newProtoTransport("3", http.DefaultTransport.(*http.Transport).Clone(), "ip", &net.Dialer{}, 0) + if err != nil { + t.Fatal(err) + } + u, _ := url.Parse(srv.URL) + pt.addServer(u.Host) + if _, err := (&http.Client{Transport: pt}).Get(srv.URL); err == nil || !strings.Contains(err.Error(), "https") { + t.Fatalf("HTTP/3 over http: got %v, want an https error", err) + } +} + func TestAutoUpgradesOnAltSvc(t *testing.T) { srv := testServer(t, true, true) c := newTestClient(t, "auto", srv) @@ -113,3 +212,66 @@ func TestAutoFallsBackWhenQUICFails(t *testing.T) { t.Fatal("forced HTTP/3 succeeded without a QUIC server") } } + +// On a network that drops UDP the handshake times out silently. auto must +// give up on it early enough to retry over TCP within the client timeout. +func TestAutoFallsBackWhenUDPIsDropped(t *testing.T) { + srv := testServer(t, false, true) + // bound but never read: packets vanish, like on a filtered network + blackhole, err := net.ListenPacket("udp", srv.Listener.Addr().String()) + if err != nil { + t.Skip(err) + } + t.Cleanup(func() { blackhole.Close() }) + c := newTestClientWith(t, "auto", srv, nil, 3*time.Second) + if got := protoOf(t, c, http.MethodGet, srv.URL); got != "HTTP/2.0" { + t.Fatalf("used %s, want fallback to HTTP/2.0", got) + } +} + +func TestAdvertisesH3(t *testing.T) { + u, _ := url.Parse("https://speed.example.com:8443/") + for header, want := range map[string]bool{ + `h3=":8443"; ma=60`: true, + `h3="speed.example.com:8443"`: true, + `h3-29=":8443", h3=":8443"; ma=86400`: true, + `h3=":8443"; ma=0`: false, + `h3=":8443"; ma="0"`: false, + `clear`: false, + `h3=":8443", clear`: false, + `h3=":443"; ma=60`: false, + `h3="other.example.com:8443"; ma=60`: false, + `h2=":8443"`: false, + `h3=:8443`: false, + `xh3=":8443"`: false, + `h3=":8443x"`: false, + } { + resp := &http.Response{Header: http.Header{"Alt-Svc": {header}}} + if got := advertisesH3(resp, u); got != want { + t.Errorf("Alt-Svc: %s: got %t, want %t", header, got, want) + } + } +} + +// What the measurement responses ran over is reported, mixed protocols too. +func TestObservedProtocols(t *testing.T) { + srv := testServer(t, true, true) + c := newTestClient(t, "auto", srv) + pt := c.Transport.(*protoTransport) + u, _ := url.Parse(srv.URL) + protoOf(t, c, http.MethodGet, srv.URL) + if proto, state, ok := pt.observedOn(u.Host); !ok || proto != "HTTP/3.0" || state == nil { + t.Fatalf("observed %q, TLS %v, %t; want HTTP/3.0 with TLS", proto, state, ok) + } + pt.resetObserved(u.Host) + if _, _, ok := pt.observedOn(u.Host); ok { + t.Fatal("observation kept after reset") + } + pt.setState(u.Host, "tcp") + protoOf(t, c, http.MethodGet, srv.URL) + pt.setState(u.Host, "h3") + protoOf(t, c, http.MethodGet, srv.URL) + if proto, _, _ := pt.observedOn(u.Host); proto != "HTTP/2.0+HTTP/3.0" { + t.Fatalf("observed %q, want HTTP/2.0+HTTP/3.0", proto) + } +} From ee2ce325e2ff9dbbf6d6247ffd25c63fa4deec44 Mon Sep 17 00:00:00 2001 From: AlphaJack <19962243+AlphaJack@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:41:30 +0100 Subject: [PATCH 3/4] fix: close QUIC sockets, retry failed uploads, sanitize network errors - Close the UDP socket of every QUIC dial: quic-go closes only the ones it opened - Retry an upload request that fails mid-test, without counting its bytes twice - Fail the upload instead of reporting 0 Mbps when nothing was sent - Sanitize network errors where the transport returns them and where they are printed - Return an error, never a nil connection, from a dial with no address - Treat ma=00 as a withdrawal, accept Clear and honour quoted Alt-Svc parameters - Accept --http-version 1 as an alias of 1.1 --- defs/bytes_counter.go | 19 ++++++ defs/server.go | 35 ++++++++++- defs/server_test.go | 56 +++++++++++++++++ main.go | 2 +- speedtest/helper.go | 12 ++-- speedtest/speedtest.go | 4 +- speedtest/transport.go | 105 +++++++++++++++++++++++++++---- speedtest/transport_test.go | 122 +++++++++++++++++++++++++++++++++++- 8 files changed, 331 insertions(+), 24 deletions(-) diff --git a/defs/bytes_counter.go b/defs/bytes_counter.go index 4d17de1..4b3e22c 100644 --- a/defs/bytes_counter.go +++ b/defs/bytes_counter.go @@ -29,6 +29,25 @@ func (c *BytesCounter) Write(p []byte) (int, error) { return n, nil } +// Discount takes back n bytes counted for a request that failed, so the +// retry that sends them again does not count them twice. +func (c *BytesCounter) Discount(n uint64) { + c.total.Add(^(n - 1)) +} + +// requestCounter counts one request's bytes into a shared counter and keeps +// its own tally, which Discount can take back if the request fails. The +// tally is atomic: the transport may still read the body after Do returns. +type requestCounter struct { + shared *BytesCounter + n atomic.Uint64 +} + +func (r *requestCounter) Write(p []byte) (int, error) { + r.n.Add(uint64(len(p))) + return r.shared.Write(p) +} + // SetBase sets the base for dividing bytes into megabyte or mebibyte func (c *BytesCounter) SetMebi(mebi bool) { c.mebi = mebi diff --git a/defs/server.go b/defs/server.go index 5391b59..3e1b5b8 100644 --- a/defs/server.go +++ b/defs/server.go @@ -19,6 +19,7 @@ import ( "slices" "strconv" "sync" + "sync/atomic" "time" "github.com/briandowns/spinner" @@ -523,6 +524,9 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int uploadDone := make(chan struct{}, requests) var wg sync.WaitGroup + // the last error of a failed request; returned if nothing was uploaded, + // so a broken path is not reported as a 0 Mbps result + var lastErr atomic.Pointer[error] doUpload := func() { defer wg.Done() @@ -533,7 +537,8 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int } else { bodyReader = bytes.NewReader(counter.Payload()) } - countingReader := io.TeeReader(bodyReader, counter) + reqCounter := &requestCounter{shared: counter} + countingReader := io.TeeReader(bodyReader, reqCounter) uploadReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), countingReader) if err != nil { @@ -559,8 +564,26 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int resp, err := http.DefaultClient.Do(uploadReq) if err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - output.WriteDebug("Failed when making HTTP request: %s\n", err) + if ctx.Err() != nil { + // cancellation is how the test ends, so it is not a failure + return + } + output.WriteDebug("Failed when making HTTP request: %s\n", err) + lastErr.Store(&err) + // The body cannot be replayed, so the transport does not retry + // it. Take back its bytes and let the main loop start a new + // request instead, after a pause so a refused connection does + // not spin. In auto mode a failed HTTP/3 request has already + // moved the server to TCP, so the new request goes over TCP. + counter.Discount(reqCounter.n.Load()) + select { + case <-time.After(uploadRetryDelay): + case <-ctx.Done(): + return + } + select { + case uploadDone <- struct{}{}: + case <-ctx.Done(): } return } @@ -637,9 +660,15 @@ Loop: // result doesn't change under us while it's being reported wg.Wait() + if err := lastErr.Load(); err != nil && counter.Total() == 0 { + return 0, 0, *err + } return counter.AvgMbps(), counter.Total(), nil } +// uploadRetryDelay spaces the retries of failed upload requests. +const uploadRetryDelay = 200 * time.Millisecond + // GetIPInfo accesses the backend's getIP.php endpoint and get current client's IP information func (s *Server) GetIPInfo(distanceUnit string) (*GetIPResult, error) { t := time.Now() diff --git a/defs/server_test.go b/defs/server_test.go index ed94c68..c7010cd 100644 --- a/defs/server_test.go +++ b/defs/server_test.go @@ -214,3 +214,59 @@ func TestUploadSendsContentLength(t *testing.T) { t.Errorf("counter reported %d bytes, want more than one payload", total) } } + +// flakyUploadServer drops the connection of the first `failures` POSTs +// without answering, then answers every POST. It returns the server and the number +// of body bytes it read from the answered POSTs. +func flakyUploadServer(t *testing.T, failures int64) (*Server, *atomic.Int64) { + t.Helper() + var seen, answered atomic.Int64 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if seen.Add(1) <= failures { + conn, _, err := http.NewResponseController(w).Hijack() + if err == nil { + conn.Close() + } + return + } + n, _ := io.Copy(io.Discard, r.Body) + answered.Add(n) + })) + t.Cleanup(ts.Close) + return &Server{Server: ts.URL, UploadURL: "/"}, &answered +} + +// A request that fails mid-test is retried instead of dropped: the body +// cannot be replayed, so the transport does not retry it, and with a single +// stream the run would otherwise upload nothing and report 0 Mbps. +func TestUploadRetriesAFailedRequest(t *testing.T) { + s, answered := flakyUploadServer(t, 1) + _, total, err := s.Upload(false, true, false, false, 1, 64, time.Second) + if err != nil { + t.Fatalf("Upload returned error: %v", err) + } + if total == 0 || answered.Load() == 0 { + t.Fatalf("uploaded %d bytes, server read %d: the failed request was not retried", total, answered.Load()) + } +} + +// When every request fails there is no measurement, and Upload says so +// instead of returning 0 Mbps as a result. +func TestUploadFailsWhenNothingIsUploaded(t *testing.T) { + s, _ := flakyUploadServer(t, 1<<30) + if _, total, err := s.Upload(false, true, false, false, 1, 64, time.Second); err == nil { + t.Fatalf("Upload of %d bytes returned no error while every request failed", total) + } +} + +// The bytes of a failed request are taken back, and only those. +func TestBytesCounterDiscount(t *testing.T) { + c := NewCounter() + c.Write(make([]byte, 100)) + r := &requestCounter{shared: c} + r.Write(make([]byte, 30)) + c.Discount(r.n.Load()) + if got := c.Total(); got != 100 { + t.Fatalf("total %d after discounting a failed request, want 100", got) + } +} diff --git a/main.go b/main.go index 84d9bb1..d1d606b 100644 --- a/main.go +++ b/main.go @@ -239,6 +239,6 @@ func main() { // run main function with cli options err := app.Run(os.Args) if err != nil { - output.Fatalf("Terminated due to error: %s", err) + output.Fatalf("Terminated due to error: %s", output.Sanitize(err.Error())) } } diff --git a/speedtest/helper.go b/speedtest/helper.go index acbc86f..c68a1bb 100644 --- a/speedtest/helper.go +++ b/speedtest/helper.go @@ -77,7 +77,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel output.WriteDebug("Fetching IP info\n") ispInfo, err := currentServer.GetIPInfo(c.String(defs.OptionDistance)) if err != nil { - output.WriteError("Failed to get IP info: %s\n", err) + output.WriteError("Failed to get IP info: %s\n", output.Sanitize(err.Error())) return err } output.WriteUI("You're testing from: %s\n", output.Sanitize(ispInfo.ProcessedString)) @@ -104,7 +104,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel p, jitter, err := currentServer.ICMPPingAndJitter(pingCount, c.String(defs.OptionSource), network) if err != nil { - output.WriteError("Failed to get ping and jitter: %s\n", err) + output.WriteError("Failed to get ping and jitter: %s\n", output.Sanitize(err.Error())) return err } @@ -137,7 +137,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel download, br, err := currentServer.Download(silent, c.Bool(defs.OptionBytes), c.Bool(defs.OptionMebiBytes), c.Int(defs.OptionConcurrent), c.Int(defs.OptionChunks), time.Duration(c.Int(defs.OptionDuration))*time.Second) if err != nil { - output.WriteError("Failed to get download speed: %s\n", err) + output.WriteError("Failed to get download speed: %s\n", output.Sanitize(err.Error())) return err } downloadValue = download @@ -159,7 +159,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel upload, bw, err := currentServer.Upload(c.Bool(defs.OptionNoPreAllocate), silent, c.Bool(defs.OptionBytes), c.Bool(defs.OptionMebiBytes), c.Int(defs.OptionConcurrent), c.Int(defs.OptionUploadSize), time.Duration(c.Int(defs.OptionDuration))*time.Second) if err != nil { - output.WriteError("Failed to get upload speed: %s\n", err) + output.WriteError("Failed to get upload speed: %s\n", output.Sanitize(err.Error())) return err } uploadValue = upload @@ -200,7 +200,7 @@ func doSpeedTest(c *cli.Context, servers []defs.Server, telemetryServer defs.Tel extra.Extra = c.String(defs.OptionTelemetryExtra) if link, err := sendTelemetry(telemetryServer, ispInfo, downloadValue, uploadValue, p, jitter, currentServer.TLog.String(), extra); err != nil { - output.WriteError("Error when sending telemetry data: %s\n", err) + output.WriteError("Error when sending telemetry data: %s\n", output.Sanitize(err.Error())) } else { shareLink = link // only print to stdout when --json and --csv are not used @@ -416,7 +416,7 @@ func sendTelemetry(telemetryServer defs.TelemetryServer, ispInfo *defs.GetIPResu id, err := io.ReadAll(resp.Body) if err != nil { - output.WriteError("Error when reading HTTP request: %s\n", err) + output.WriteError("Error when reading HTTP request: %s\n", output.Sanitize(err.Error())) return "", err } diff --git a/speedtest/speedtest.go b/speedtest/speedtest.go index ac7f129..04f6e7b 100644 --- a/speedtest/speedtest.go +++ b/speedtest/speedtest.go @@ -259,7 +259,7 @@ func SpeedTest(c *cli.Context) error { http.DefaultClient.Transport = pt activeTransport = pt // telemetry stays on TCP, even when it shares a host with a test server - telemetryClient = &http.Client{Transport: pt.base, Timeout: http.DefaultClient.Timeout} + telemetryClient = &http.Client{Transport: sanitizingTransport{pt.base}, Timeout: http.DefaultClient.Timeout} // no scheme is forced by default // force https if --secure is given @@ -300,7 +300,7 @@ func SpeedTest(c *cli.Context) error { } } if err != nil { - output.WriteError("Error when fetching server list: %s\n", err) + output.WriteError("Error when fetching server list: %s\n", output.Sanitize(err.Error())) return err } for _, s := range servers { diff --git a/speedtest/transport.go b/speedtest/transport.go index 2cd0903..aa5fba3 100644 --- a/speedtest/transport.go +++ b/speedtest/transport.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "errors" "fmt" + "io" "net" "net/http" "net/url" @@ -52,7 +53,7 @@ func newProtoTransport(version string, tcp *http.Transport, network string, dial t := &protoTransport{version: version, tcp: tcp, hosts: map[string]string{}, observed: map[string]*observation{}} switch version { case "auto", "3": - case "1.1": + case "1", "1.1": t.tcp.Protocols = new(http.Protocols) t.tcp.Protocols.SetHTTP1(true) case "2": @@ -105,15 +106,58 @@ func (t *protoTransport) RoundTrip(req *http.Request) (*http.Response, error) { state, isServer := t.hosts[req.URL.Host] t.mu.Unlock() if !isServer { - return t.base.RoundTrip(req) + return sanitizingTransport{t.base}.RoundTrip(req) } - resp, err := t.roundTrip(req, state) + resp, err := sanitizingTransport{roundTripFunc(func(req *http.Request) (*http.Response, error) { + return t.roundTrip(req, state) + })}.RoundTrip(req) if err == nil { t.observe(req.URL.Host, resp) } return resp, err } +// roundTripFunc lets a function serve as an http.RoundTripper. +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +// sanitizingTransport cleans the errors of rt and of the bodies it returns. +type sanitizingTransport struct{ rt http.RoundTripper } + +func (s sanitizingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := s.rt.RoundTrip(req) + if err != nil { + return nil, sanitizedError{err} + } + resp.Body = sanitizedBody{resp.Body} + return resp, nil +} + +// sanitizedError hides control characters in a network error, which can +// carry text from the server (an HTTP/3 close reason, a certificate name). +// Callers print it, so it is cleaned where every request passes. Unwrap +// keeps errors.Is working, for context.Canceled in particular. +type sanitizedError struct{ err error } + +func (e sanitizedError) Error() string { + // errors.Join puts each address's error on its own line + return output.Sanitize(strings.ReplaceAll(e.err.Error(), "\n", "; ")) +} + +func (e sanitizedError) Unwrap() error { return e.err } + +// sanitizedBody does the same for errors met while reading the body. +type sanitizedBody struct{ io.ReadCloser } + +func (b sanitizedBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + if err != nil && err != io.EOF { + err = sanitizedError{err} + } + return n, err +} + func (t *protoTransport) roundTrip(req *http.Request, state string) (*http.Response, error) { switch { case t.version == "3": @@ -192,9 +236,7 @@ func proxyError(proxy string, err error) error { // h3Failure describes a failed HTTP/3 request. err can carry text from the // server, such as certificate names, so it is sanitized like the host. func h3Failure(host string, err error) string { - // errors.Join puts each address's error on its own line - msg := strings.ReplaceAll(err.Error(), "\n", "; ") - return fmt.Sprintf("HTTP/3 to %s failed: %s", output.Sanitize(host), output.Sanitize(msg)) + return fmt.Sprintf("HTTP/3 to %s failed: %s", output.Sanitize(host), sanitizedError{err}) } func (t *protoTransport) setState(host, state string) { @@ -261,12 +303,14 @@ type altSvc struct { func parseAltSvc(values []string) []altSvc { var alts []altSvc for _, v := range values { - for entry := range strings.SplitSeq(v, ",") { + for _, entry := range splitUnquoted(v, ',') { entry = strings.TrimSpace(entry) - if entry == "clear" { + // RFC 7838 spells it in lower case; any case is taken here, as + // staying on TCP is the safe reading + if strings.EqualFold(entry, "clear") { return nil } - params := strings.Split(entry, ";") + params := splitUnquoted(entry, ';') protocol, authority, ok := strings.Cut(params[0], "=") if !ok { continue @@ -285,7 +329,15 @@ func parseAltSvc(values []string) []altSvc { withdrawn := false for _, param := range params[1:] { name, value, _ := strings.Cut(strings.TrimSpace(param), "=") - if strings.EqualFold(strings.TrimSpace(name), "ma") && strings.Trim(strings.TrimSpace(value), `"`) == "0" { + if !strings.EqualFold(strings.TrimSpace(name), "ma") { + continue + } + value = strings.TrimSpace(value) + if unquoted, err := strconv.Unquote(value); err == nil { + value = unquoted + } + // delta-seconds: ma=00 withdraws the alternative like ma=0 + if ma, err := strconv.ParseUint(value, 10, 64); err == nil && ma == 0 { withdrawn = true } } @@ -297,6 +349,27 @@ func parseAltSvc(values []string) []altSvc { return alts } +// splitUnquoted splits s at each sep outside a quoted string, so a quoted +// parameter value cannot end an entry or start a new parameter. +func splitUnquoted(s string, sep byte) []string { + var parts []string + quoted, escaped, start := false, false, 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case escaped: + escaped = false + case quoted && c == '\\': + escaped = true + case c == '"': + quoted = !quoted + case !quoted && c == sep: + parts = append(parts, s[start:i]) + start = i + 1 + } + } + return append(parts, s[start:]) +} + func replayable(req *http.Request) bool { return req.Method == http.MethodGet && (req.Body == nil || req.Body == http.NoBody) } @@ -312,6 +385,9 @@ func dialQUIC(ctx context.Context, network string, dialer *net.Dialer, addr stri if err != nil { return nil, err } + if len(ips) == 0 { + return nil, fmt.Errorf("no address found for %s", host) + } return dialQUICAddrs(ctx, ips, port, dialer, tlsConf, quicConf) } @@ -343,18 +419,25 @@ func dialQUICAddrs(ctx context.Context, ips []net.IP, port string, dialer *net.D errs = append(errs, fmt.Errorf("%s: %w", ip, err)) continue } + // quic-go closes only the sockets it opened itself, so pc is closed + // here after the transport, on both paths tr := &quic.Transport{Conn: pc} conn, err := tr.DialEarly(ctx, raddr, tlsConf, quicConf) if err == nil { - go func() { <-conn.Context().Done(); tr.Close() }() + go func() { <-conn.Context().Done(); tr.Close(); pc.Close() }() return conn, nil } tr.Close() + pc.Close() errs = append(errs, fmt.Errorf("%s: %w", ip, err)) if ctx.Err() != nil { break } } + if len(errs) == 0 { + // never a nil connection with a nil error: http3 would dereference it + return nil, errors.New("no address to dial") + } return nil, errors.Join(errs...) } diff --git a/speedtest/transport_test.go b/speedtest/transport_test.go index c455142..99ef665 100644 --- a/speedtest/transport_test.go +++ b/speedtest/transport_test.go @@ -5,14 +5,18 @@ import ( "crypto/tls" "crypto/x509" "errors" + "fmt" "io" "net" "net/http" "net/http/httptest" "net/url" + "os" + "runtime/debug" "strings" "sync/atomic" "testing" + "testing/iotest" "time" "github.com/quic-go/quic-go/http3" @@ -86,9 +90,11 @@ func protoOf(t *testing.T, c *http.Client, method, url string) string { return resp.Proto } +// Each forced version is used as asked, and auto stays on TCP while the +// server does not advertise HTTP/3. func TestHTTPVersions(t *testing.T) { srv := testServer(t, true, false) - for version, want := range map[string]string{"1.1": "HTTP/1.1", "2": "HTTP/2.0", "3": "HTTP/3.0", "auto": "HTTP/2.0"} { + for version, want := range map[string]string{"1": "HTTP/1.1", "1.1": "HTTP/1.1", "2": "HTTP/2.0", "3": "HTTP/3.0", "auto": "HTTP/2.0"} { t.Run(version, func(t *testing.T) { if got := protoOf(t, newTestClient(t, version, srv), http.MethodGet, srv.URL); got != want { t.Errorf("used %s, want %s", got, want) @@ -166,6 +172,8 @@ func TestDialQUICSkipsOtherFamily(t *testing.T) { conn.CloseWithError(0, "") } +// A failure message names the host and the cause without their control +// characters. func TestH3FailureIsSanitized(t *testing.T) { msg := h3Failure("host\x1b[2J", errors.New("x509: certificate is valid for \x1b[2Jevil\n")) if strings.ContainsAny(msg, "\x1b\n") { @@ -173,6 +181,7 @@ func TestH3FailureIsSanitized(t *testing.T) { } } +// HTTP/3 to an http:// server fails with a hint to use https. func TestHTTP3NeedsHTTPS(t *testing.T) { srv := httptest.NewServer(http.NotFoundHandler()) t.Cleanup(srv.Close) @@ -187,6 +196,8 @@ func TestHTTP3NeedsHTTPS(t *testing.T) { } } +// A server that advertises HTTP/3 is reached over it from the first +// request on, uploads included. func TestAutoUpgradesOnAltSvc(t *testing.T) { srv := testServer(t, true, true) c := newTestClient(t, "auto", srv) @@ -198,6 +209,8 @@ func TestAutoUpgradesOnAltSvc(t *testing.T) { } } +// A server that advertises HTTP/3 without serving it is reached over TCP in +// auto mode, and fails a forced HTTP/3. func TestAutoFallsBackWhenQUICFails(t *testing.T) { srv := testServer(t, false, true) c := newTestClient(t, "auto", srv) @@ -229,6 +242,8 @@ func TestAutoFallsBackWhenUDPIsDropped(t *testing.T) { } } +// Only an h3 alternative on the same host and port counts, and ma=0 or +// clear withdraws it. func TestAdvertisesH3(t *testing.T) { u, _ := url.Parse("https://speed.example.com:8443/") for header, want := range map[string]bool{ @@ -245,6 +260,12 @@ func TestAdvertisesH3(t *testing.T) { `h3=:8443`: false, `xh3=":8443"`: false, `h3=":8443x"`: false, + `h3=":8443"; ma=00`: false, + `h3=":8443"; ma="00"`: false, + `Clear`: false, + `h3=":8443", Clear`: false, + `h3=":8443"; note="x;ma=0"`: true, + `h3=":8443"; note="a,clear"`: true, } { resp := &http.Response{Header: http.Header{"Alt-Svc": {header}}} if got := advertisesH3(resp, u); got != want { @@ -275,3 +296,102 @@ func TestObservedProtocols(t *testing.T) { t.Fatalf("observed %q, want HTTP/2.0+HTTP/3.0", proto) } } + +// errors.Join of no errors is nil, and http3 dereferences the connection of +// a dial that returns a nil error. +func TestDialQUICWithoutAddresses(t *testing.T) { + conn, err := dialQUICAddrs(context.Background(), nil, "443", &net.Dialer{}, &tls.Config{}, nil) + if conn != nil || err == nil { + t.Fatalf("got conn %v, error %v; want an error", conn, err) + } +} + +// openFDs counts this process's open file descriptors. +func openFDs(t *testing.T) int { + t.Helper() + fds, err := os.ReadDir("/dev/fd") + if err != nil { + t.Skip(err) + } + return len(fds) +} + +// Every UDP socket a dial opens is closed again, whether the dial fails or +// the connection it made ends. GC is off so finalizers cannot hide a leak. +func TestDialQUICClosesItsSocket(t *testing.T) { + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + srv := testServer(t, true, false) + _, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + // a port with nothing behind it, for the failing dials + closed, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Skip(err) + } + _, closedPort, _ := net.SplitHostPort(closed.LocalAddr().String()) + closed.Close() + + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + tlsConf := &tls.Config{RootCAs: pool, ServerName: "example.com", NextProtos: []string{http3.NextProtoH3}} + loopback := []net.IP{net.ParseIP("127.0.0.1")} + + before := openFDs(t) + for range 10 { + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + if conn, err := dialQUICAddrs(ctx, loopback, closedPort, &net.Dialer{}, tlsConf, nil); err == nil { + conn.CloseWithError(0, "") + t.Fatal("dial to a closed port succeeded") + } + cancel() + conn, err := dialQUICAddrs(context.Background(), loopback, port, &net.Dialer{}, tlsConf, nil) + if err != nil { + t.Fatal(err) + } + conn.CloseWithError(0, "") + } + // the socket of a closed connection is closed in the background + deadline := time.Now().Add(2 * time.Second) + for openFDs(t) > before+2 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if after := openFDs(t); after > before+2 { + t.Fatalf("open file descriptors went from %d to %d after 20 dials", before, after) + } +} + +// Errors and body read errors from a test server reach the caller with +// their control characters removed, as every caller prints them. +func TestTransportSanitizesErrors(t *testing.T) { + srv := testServer(t, false, false) + c := newTestClient(t, "2", srv) + pt := c.Transport.(*protoTransport) + pt.tcp.DialContext = func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("boom \x1b[2J forged\nline") + } + pt.tcp.DialTLSContext = nil + _, err := c.Get(srv.URL) + if err == nil || strings.ContainsAny(err.Error(), "\x1b\n") { + t.Fatalf("got %q, want an error without control characters", err) + } + + // the server list and telemetry go through the same cleaning + other := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(other.Close) + pt.base.DialContext = pt.tcp.DialContext + if _, err := c.Get(other.URL); err == nil || strings.ContainsAny(err.Error(), "\x1b\n") { + t.Fatalf("server list request: got %q, want an error without control characters", err) + } + + cancelled := sanitizedError{fmt.Errorf("wrapped: %w", context.Canceled)} + if !errors.Is(cancelled, context.Canceled) { + t.Error("sanitizing hides context.Canceled") + } + body := sanitizedBody{io.NopCloser(iotest.ErrReader(errors.New("reset \x1b[2J")))} + if _, err := body.Read(make([]byte, 1)); err == nil || strings.Contains(err.Error(), "\x1b") { + t.Errorf("body read error %q kept its control characters", err) + } + body = sanitizedBody{io.NopCloser(strings.NewReader(""))} + if _, err := body.Read(make([]byte, 1)); err != io.EOF { + t.Errorf("body read at the end returned %v, want io.EOF itself", err) + } +} From 99ac31c79f21c69c174aa0e2c43aebe02061ebd6 Mon Sep 17 00:00:00 2001 From: AlphaJack <19962243+AlphaJack@users.noreply.github.com> Date: Sun, 22 Nov 2026 17:53:03 +0100 Subject: [PATCH 4/4] fix: count the bytes of upload requests that fail With --no-pre-allocate the body never ends, so every request ends at --timeout. Taking back the bytes of failed requests then dropped nearly everything the server received: 0.49 GB reported for 22.7 GB received. A retry sends new bytes, so nothing is counted twice. The upload now fails only when no request got an answer and the last error was not the client timeout, which means the body was still streaming. --- defs/bytes_counter.go | 19 ------------------- defs/server.go | 30 ++++++++++++++++++++---------- defs/server_test.go | 31 ++++++++++++++++++++++--------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/defs/bytes_counter.go b/defs/bytes_counter.go index 4b3e22c..4d17de1 100644 --- a/defs/bytes_counter.go +++ b/defs/bytes_counter.go @@ -29,25 +29,6 @@ func (c *BytesCounter) Write(p []byte) (int, error) { return n, nil } -// Discount takes back n bytes counted for a request that failed, so the -// retry that sends them again does not count them twice. -func (c *BytesCounter) Discount(n uint64) { - c.total.Add(^(n - 1)) -} - -// requestCounter counts one request's bytes into a shared counter and keeps -// its own tally, which Discount can take back if the request fails. The -// tally is atomic: the transport may still read the body after Do returns. -type requestCounter struct { - shared *BytesCounter - n atomic.Uint64 -} - -func (r *requestCounter) Write(p []byte) (int, error) { - r.n.Add(uint64(len(p))) - return r.shared.Write(p) -} - // SetBase sets the base for dividing bytes into megabyte or mebibyte func (c *BytesCounter) SetMebi(mebi bool) { c.mebi = mebi diff --git a/defs/server.go b/defs/server.go index 3e1b5b8..5381025 100644 --- a/defs/server.go +++ b/defs/server.go @@ -524,9 +524,10 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int uploadDone := make(chan struct{}, requests) var wg sync.WaitGroup - // the last error of a failed request; returned if nothing was uploaded, - // so a broken path is not reported as a 0 Mbps result + // the last error of a failed request, returned when no request got an + // answer, so a broken path is not reported as a 0 Mbps result var lastErr atomic.Pointer[error] + var answered atomic.Bool doUpload := func() { defer wg.Done() @@ -537,8 +538,7 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int } else { bodyReader = bytes.NewReader(counter.Payload()) } - reqCounter := &requestCounter{shared: counter} - countingReader := io.TeeReader(bodyReader, reqCounter) + countingReader := io.TeeReader(bodyReader, counter) uploadReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), countingReader) if err != nil { @@ -571,11 +571,12 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int output.WriteDebug("Failed when making HTTP request: %s\n", err) lastErr.Store(&err) // The body cannot be replayed, so the transport does not retry - // it. Take back its bytes and let the main loop start a new - // request instead, after a pause so a refused connection does - // not spin. In auto mode a failed HTTP/3 request has already - // moved the server to TCP, so the new request goes over TCP. - counter.Discount(reqCounter.n.Load()) + // it. Let the main loop start a new request instead, after a + // pause so a refused connection does not spin. In auto mode a + // failed HTTP/3 request has already moved the server to TCP, so + // the new request goes over TCP. The bytes this one sent stay + // counted: with --no-pre-allocate the body never ends, every + // request ends at --timeout, and the server got all of them. select { case <-time.After(uploadRetryDelay): case <-ctx.Done(): @@ -588,6 +589,7 @@ func (s *Server) Upload(noPrealloc, silent, useBytes, useMebi bool, requests int return } defer resp.Body.Close() + answered.Store(true) if _, err := io.Copy(io.Discard, resp.Body); err != nil { // cancellation is how the test ends, so it is not a failure @@ -660,12 +662,20 @@ Loop: // result doesn't change under us while it's being reported wg.Wait() - if err := lastErr.Load(); err != nil && counter.Total() == 0 { + // A request that ran into the client timeout was still sending, as with + // --no-pre-allocate where the body never ends, so its bytes count as a + // result. Any other failure with no answer at all is a broken path. + if err := lastErr.Load(); err != nil && !answered.Load() && !isTimeout(*err) { return 0, 0, *err } return counter.AvgMbps(), counter.Total(), nil } +func isTimeout(err error) bool { + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + // uploadRetryDelay spaces the retries of failed upload requests. const uploadRetryDelay = 200 * time.Millisecond diff --git a/defs/server_test.go b/defs/server_test.go index c7010cd..0fd2a47 100644 --- a/defs/server_test.go +++ b/defs/server_test.go @@ -259,14 +259,27 @@ func TestUploadFailsWhenNothingIsUploaded(t *testing.T) { } } -// The bytes of a failed request are taken back, and only those. -func TestBytesCounterDiscount(t *testing.T) { - c := NewCounter() - c.Write(make([]byte, 100)) - r := &requestCounter{shared: c} - r.Write(make([]byte, 30)) - c.Discount(r.n.Load()) - if got := c.Total(); got != 100 { - t.Fatalf("total %d after discounting a failed request, want 100", got) +// TestUploadCountsRequestsThatTimeOut: with --no-pre-allocate the body never +// ends, so every request ends at the client timeout. The server still got +// what was sent, and the result must count it. +func TestUploadCountsRequestsThatTimeOut(t *testing.T) { + var received atomic.Int64 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n, _ := io.Copy(io.Discard, r.Body) + received.Add(n) + })) + t.Cleanup(ts.Close) + defer func(old time.Duration) { http.DefaultClient.Timeout = old }(http.DefaultClient.Timeout) + http.DefaultClient.Timeout = 300 * time.Millisecond + + s := &Server{Server: ts.URL, UploadURL: "/"} + _, total, err := s.Upload(true, true, false, false, 1, 64, 1500*time.Millisecond) + if err != nil { + t.Fatalf("Upload returned error: %v", err) + } + // let the server finish counting the last request + time.Sleep(100 * time.Millisecond) + if got := received.Load(); total < uint64(got)/2 { + t.Fatalf("reported %d bytes while the server received %d", total, got) } }