diff --git a/gateway/Caddyfile.tmpl b/gateway/Caddyfile.tmpl index 0263773b9a5..95651ae8f0e 100644 --- a/gateway/Caddyfile.tmpl +++ b/gateway/Caddyfile.tmpl @@ -241,12 +241,20 @@ http://healthcheck.internal { } {{- end }} + @address vars_regexp {http.request.host} (?i)^[0-9a-f]{32}\. + # The path belongs to the device's own HTTP server and is not ours to # interpret. Caddy forwards it as the client wrote it, escaping intact. - reverse_proxy {{ $cfg.APIBackend }} { - header_up Host {host} - header_up X-Request-ID {http.request.uuid} - header_up X-Real-IP {remote_host} + handle @address { + reverse_proxy {{ $cfg.APIBackend }} { + header_up Host {host} + header_up X-Request-ID {http.request.uuid} + header_up X-Real-IP {remote_host} + } + } + + handle { + respond 404 } } {{- end }} diff --git a/openapi/spec/paths/api@namespaces@{tenant}.yaml b/openapi/spec/paths/api@namespaces@{tenant}.yaml index d3711977d85..c7fd7965c4d 100644 --- a/openapi/spec/paths/api@namespaces@{tenant}.yaml +++ b/openapi/spec/paths/api@namespaces@{tenant}.yaml @@ -73,6 +73,8 @@ delete: description: Success to delete a namespace. '401': $ref: ../components/responses/401.yaml + '402': + $ref: ../components/responses/402.yaml '403': $ref: ../components/responses/403.yaml '404': diff --git a/pkg/webendpoints/webendpoints.go b/pkg/webendpoints/webendpoints.go index ec46221744c..d99f3a7d5c9 100644 --- a/pkg/webendpoints/webendpoints.go +++ b/pkg/webendpoints/webendpoints.go @@ -36,6 +36,10 @@ func Domain(preferred, fallback string) string { return fallback } +// ProxyPath is the server route a request addressed to a web endpoint is rewritten to before it +// is carried into the device. The response is the device's own, so no API schema describes it. +const ProxyPath = "/http/proxy" + var address = regexp.MustCompile(`^[a-f0-9]{32}$`) // AddressFromHost is the inverse of Host: it takes the Host of an inbound diff --git a/server/app/openapi_test.go b/server/app/openapi_test.go index d78f69b62de..6d92ec5cbd4 100644 --- a/server/app/openapi_test.go +++ b/server/app/openapi_test.go @@ -9,6 +9,7 @@ import ( "github.com/gorilla/websocket" "github.com/labstack/echo/v5" + "github.com/shellhub-io/shellhub/pkg/webendpoints" "github.com/shellhub-io/shellhub/server/api/routes" "github.com/shellhub-io/shellhub/server/api/routes/middleware" sshhttp "github.com/shellhub-io/shellhub/server/ssh/http" @@ -122,6 +123,7 @@ func TestOpenAPIValidationSkipper(t *testing.T) { {routes.InternalMetricsURL, true}, {"/mcp", true}, {"/mcp/session", true}, + {webendpoints.ProxyPath, true}, {web.WebSessionRoute, false}, {"/api/devices", false}, {"/api/namespaces", false}, diff --git a/server/app/server.go b/server/app/server.go index e5365c7b0ed..ea85f3e4b24 100644 --- a/server/app/server.go +++ b/server/app/server.go @@ -17,6 +17,7 @@ import ( "github.com/shellhub-io/shellhub/pkg/api/query" "github.com/shellhub-io/shellhub/pkg/cache" "github.com/shellhub-io/shellhub/pkg/envs" + "github.com/shellhub-io/shellhub/pkg/webendpoints" "github.com/shellhub-io/shellhub/pkg/worker" "github.com/shellhub-io/shellhub/pkg/worker/asynq" "github.com/shellhub-io/shellhub/server/api/routes" @@ -447,7 +448,8 @@ func openAPIValidationSkipper(ctx *echo.Context) bool { case sshhttp.HandleConnectionV1Path, sshhttp.HandleConnectionV2Path, sshhttp.HandleRevdialPath, - web.WebsocketSSHBridgeRoute: + web.WebsocketSSHBridgeRoute, + webendpoints.ProxyPath: return true } diff --git a/tests/cmd/device-http/Dockerfile b/tests/cmd/device-http/Dockerfile new file mode 100644 index 00000000000..a0069e09db9 --- /dev/null +++ b/tests/cmd/device-http/Dockerfile @@ -0,0 +1,13 @@ +ARG GOLANG_VERSION + +FROM golang:${GOLANG_VERSION} + +WORKDIR /src + +COPY main.go . + +RUN CGO_ENABLED=0 go build -o /usr/local/bin/device-http main.go + +USER nobody + +ENTRYPOINT ["device-http"] diff --git a/tests/cmd/device-http/main.go b/tests/cmd/device-http/main.go new file mode 100644 index 00000000000..4f7eac44777 --- /dev/null +++ b/tests/cmd/device-http/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "log" + "math/big" + "net" + "net/http" + "os" + "time" +) + +type seen struct { + Name string `json:"name"` + Host string `json:"host"` + URI string `json:"uri"` + TLS bool `json:"tls"` +} + +func selfSigned(domain string) (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, err + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: domain}, + DNSNames: []string{domain}, + NotBefore: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), + NotAfter: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, err + } + + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, nil +} + +func main() { + name := os.Getenv("NAME") + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(seen{Name: name, Host: r.Host, URI: r.RequestURI, TLS: r.TLS != nil}); err != nil { + log.Printf("answering a request: %v", err) + } + }) + + certificate, err := selfSigned(os.Getenv("TLS_DOMAIN")) + if err != nil { + log.Fatal(err) + } + + var listen net.ListenConfig + + plainListener, err := listen.Listen(context.Background(), "tcp", os.Getenv("HTTP_ADDRESS")) + if err != nil { + log.Fatal(err) + } + + tcpListener, err := listen.Listen(context.Background(), "tcp", os.Getenv("TLS_ADDRESS")) + if err != nil { + log.Fatal(err) + } + + secureListener := tls.NewListener(tcpListener, &tls.Config{ + Certificates: []tls.Certificate{certificate}, + MinVersion: tls.VersionTLS13, + }) + + plain := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second} + secure := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second} + + go func() { log.Fatal(secure.Serve(secureListener)) }() + + log.Print("listening") + log.Fatal(plain.Serve(plainListener)) +} diff --git a/tests/environment/configurator.go b/tests/environment/configurator.go index 0ed514782ff..ae4563da0bd 100644 --- a/tests/environment/configurator.go +++ b/tests/environment/configurator.go @@ -82,6 +82,17 @@ func (dcc *DockerComposeConfigurator) WithEveryAddressIn(country string) *Docker return dcc } +// WithAutoSSL serves the gateway over HTTPS too, on a port reserved for it, which +// [DockerCompose.HTTPSAddress] names. The gateway then redirects a plain HTTP request to the +// instance's domain to HTTPS on port 443, which the stack does not publish, so a client of +// [DockerCompose.R] reaches no API route. +func (dcc *DockerComposeConfigurator) WithAutoSSL() *DockerComposeConfigurator { + dcc.cfg.AutoSSL = true + dcc.cfg.HTTPSPort = ReservePort(dcc.t) + + return dcc +} + // WithCronTrigger publishes the stack's redis on a port reserved for it, so [DockerCompose.RunCron] // can reach the queue the server schedules its cron jobs on. Without it redis takes whatever port // the daemon picks, which the test neither knows nor, under rootless Docker, can reach. diff --git a/tests/environment/docker_compose.go b/tests/environment/docker_compose.go index 56b9858fcd4..91c2c77ef60 100644 --- a/tests/environment/docker_compose.go +++ b/tests/environment/docker_compose.go @@ -56,6 +56,12 @@ func (dc *DockerCompose) SSHAddress() string { return dc.stack.SSHAddress() } +// HTTPSAddress is the host address the gateway's HTTPS port is published on. Only a stack brought +// up with [DockerComposeConfigurator.WithAutoSSL] publishes it. +func (dc *DockerCompose) HTTPSAddress() string { + return dc.stack.HTTPSAddress() +} + // BaseURL is the HTTP base URL the gateway is published on, for a client [DockerCompose.R] cannot // stand in for, such as a WebSocket. func (dc *DockerCompose) BaseURL() string { diff --git a/tests/environment/stack.go b/tests/environment/stack.go index 5d30829a283..d6c688194a6 100644 --- a/tests/environment/stack.go +++ b/tests/environment/stack.go @@ -42,7 +42,10 @@ import ( // HTTPSPort, random when empty. SuppliedCertificate is the certificate the gateway serves instead of // obtaining one. ACME starts a Pebble ACME server in the stack and points the gateway's ACME CA at // it. Pebble validates every challenge without reaching the gateway, so SHELLHUB_DOMAIN only has to -// be a name the gateway asks a public CA for, which localhost is not. +// be a name the gateway asks a public CA for, which localhost is not. AutoSSL turns the gateway's +// automatic HTTPS on with neither, so the gateway gets its certificates on its own; the gateway then +// redirects a plain HTTP request to the instance's domain to HTTPS on port 443, which the stack +// does not publish. type Config struct { Edition Edition Name string @@ -58,6 +61,7 @@ type Config struct { LocatedCountry string SuppliedCertificate *Certificate ACME bool + AutoSSL bool } type imageBuild struct { @@ -130,7 +134,7 @@ func Up(ctx context.Context, cfg Config) (_ *Stack, err error) { } ports := []*string{&cfg.HTTPPort, &cfg.SSHPort} - if cfg.SuppliedCertificate != nil || cfg.ACME { + if cfg.SuppliedCertificate != nil || cfg.ACME || cfg.AutoSSL { ports = append(ports, &cfg.HTTPSPort) } @@ -394,7 +398,7 @@ func (s *Stack) SSHPort() string { return s.envs["SHELLHUB_SSH_PORT"] } func (s *Stack) BaseURL() string { return "http://localhost:" + s.HTTPPort() } // HTTPSAddress returns the host:port the gateway publishes HTTPS on. Only a stack started with -// [Config.SuppliedCertificate] or [Config.ACME] publishes it. +// [Config.SuppliedCertificate], [Config.ACME] or [Config.AutoSSL] publishes it. func (s *Stack) HTTPSAddress() string { return "localhost:" + s.envs["SHELLHUB_HTTPS_PORT"] } // SSHAddress returns the host:port the gateway's SSH listener is reachable at. diff --git a/tests/environment/tls.go b/tests/environment/tls.go index 14d87d66010..bd71988336b 100644 --- a/tests/environment/tls.go +++ b/tests/environment/tls.go @@ -111,6 +111,8 @@ func (cfg Config) httpsLayer() (httpsLayer, error) { layer, err = cfg.suppliedCertificateLayer() case cfg.ACME: layer, err = cfg.acmeLayer() + case cfg.AutoSSL: + layer = httpsLayer{envs: map[string]string{}, composeFile: "../docker-compose.autossl.yml"} default: return httpsLayer{}, nil } diff --git a/tests/environment/web_endpoint.go b/tests/environment/web_endpoint.go new file mode 100644 index 00000000000..015d0846ca8 --- /dev/null +++ b/tests/environment/web_endpoint.go @@ -0,0 +1,206 @@ +package environment + +import ( + "context" + "crypto/tls" + "errors" + "io" + "net/http" + "testing" + "time" + + "github.com/go-resty/resty/v2" + "github.com/stretchr/testify/require" +) + +// WebEndpointTLS is how a web endpoint reaches the service on the device: over TLS or not, whether +// it verifies the device's certificate, and the name it verifies it against and sends as Host. +type WebEndpointTLS struct { + Enabled bool `json:"enabled"` + Verify bool `json:"verify"` + Domain string `json:"domain"` +} + +// WebEndpointCreate is the body of POST /api/web-endpoints. TTL is in seconds, -1 for an endpoint +// that never expires. +type WebEndpointCreate struct { + UID string `json:"uid"` + Host string `json:"host"` + Port int `json:"port"` + TTL int `json:"ttl"` + TLS *WebEndpointTLS `json:"tls,omitempty"` +} + +// WebEndpoint is a web endpoint as the enterprise API answers it. Address is the label the endpoint +// answers under, FullAddress that label under the instance's web endpoint domain. ExpiresIn is the +// zero time for an endpoint that never expires. +type WebEndpoint struct { + Address string `json:"address"` + FullAddress string `json:"full_address"` + Namespace string `json:"namespace"` + DeviceUID string `json:"device_uid"` + Host string `json:"host"` + Port int `json:"port"` + TTL int `json:"ttl"` + TLS WebEndpointTLS `json:"tls"` + ExpiresIn time.Time `json:"expires_in"` + CreatedAt time.Time `json:"created_at"` +} + +// TunnelCreate is the body of the deprecated POST /api/devices/:uid/tunnels. +type TunnelCreate struct { + Host string `json:"host"` + Port int `json:"port"` + TTL int `json:"ttl"` +} + +// Tunnel is a web endpoint as the deprecated device tunnel API lists it, naming its device's UID +// in Device. +type Tunnel struct { + Address string `json:"address"` + FullAddress string `json:"full_address"` + Namespace string `json:"namespace"` + Device string `json:"device"` + Host string `json:"host"` + Port int `json:"port"` + TTL int `json:"ttl"` + ExpiresIn time.Time `json:"expires_in"` + CreatedAt time.Time `json:"created_at"` +} + +// PostWebEndpoint sends body to POST /api/web-endpoints as the bearer [DockerCompose.R] carries and +// returns the answer whatever its status code, decoding a 200 into result when result is not nil. +// ctx bounds the request. It returns the error of a request that never got an answer, or whose +// answer could not be read or decoded. +func (dc *DockerCompose) PostWebEndpoint(ctx context.Context, body WebEndpointCreate, result *WebEndpoint) (*resty.Response, error) { + req := dc.R(ctx).SetBody(body) + if result != nil { + req = req.SetResult(result) + } + + return req.Post("/api/web-endpoints") +} + +// CreateWebEndpoint creates a web endpoint in the authenticated namespace and returns it as the +// server answered, failing t unless the server answers 200. It deletes the endpoint when t ends, +// ignoring the answer, so a test that already deleted it changes nothing. +func (dc *DockerCompose) CreateWebEndpoint(t *testing.T, req WebEndpointCreate) WebEndpoint { + t.Helper() + + endpoint := WebEndpoint{} + + resp, err := dc.PostWebEndpoint(t.Context(), req, &endpoint) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + t.Cleanup(func() { + _, _ = dc.DeleteWebEndpoint(context.WithoutCancel(t.Context()), endpoint.Address) + }) + + return endpoint +} + +// ListWebEndpoints returns the authenticated namespace's web endpoints, expired ones included, a +// single page of the maximum size, failing t unless the server answers 200. +func (dc *DockerCompose) ListWebEndpoints(t *testing.T) []WebEndpoint { + t.Helper() + + endpoints := []WebEndpoint{} + + resp, err := dc.R(t.Context()). + SetQueryParam("per_page", "100"). + SetResult(&endpoints). + Get("/api/web-endpoints") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + return endpoints +} + +// DeleteWebEndpoint asks to remove the web endpoint address as the bearer [DockerCompose.R] carries +// and returns the answer whatever its status code. ctx bounds the request. It returns the error of +// a request that never got an answer or whose answer could not be read. +func (dc *DockerCompose) DeleteWebEndpoint(ctx context.Context, address string) (*resty.Response, error) { + return dc.R(ctx).Delete("/api/web-endpoints/" + address) +} + +// PostTunnel sends body to the deprecated POST /api/devices/:uid/tunnels as the bearer +// [DockerCompose.R] carries and returns the answer whatever its status code. ctx bounds the +// request. It returns the error of a request that never got an answer or whose answer could not be +// read. +func (dc *DockerCompose) PostTunnel(ctx context.Context, uid string, body TunnelCreate) (*resty.Response, error) { + return dc.R(ctx).SetBody(body).Post("/api/devices/" + uid + "/tunnels") +} + +// ListTunnels returns the web endpoints of the device uid through the deprecated tunnel API, a +// single page of the maximum size, failing t unless the server answers 200. +func (dc *DockerCompose) ListTunnels(t *testing.T, uid string) []Tunnel { + t.Helper() + + tunnels := []Tunnel{} + + resp, err := dc.R(t.Context()). + SetQueryParam("per_page", "100"). + SetResult(&tunnels). + Get("/api/devices/" + uid + "/tunnels") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + return tunnels +} + +// DeleteTunnel asks the deprecated tunnel API to remove the web endpoint address of the device uid +// as the bearer [DockerCompose.R] carries and returns the answer whatever its status code. ctx +// bounds the request. It returns the error of a request that never got an answer or whose answer +// could not be read. +func (dc *DockerCompose) DeleteTunnel(ctx context.Context, uid, address string) (*resty.Response, error) { + return dc.R(ctx).Delete("/api/devices/" + uid + "/tunnels/" + address) +} + +// WebEndpointResponse is what the gateway answered a request sent to a web endpoint's address. +type WebEndpointResponse struct { + StatusCode int + Body string +} + +// VisitWebEndpoint sends GET uri to the gateway's HTTP port under the Host host, as a browser that +// resolved a web endpoint's address to the gateway does, carrying no credential, on a connection +// of its own that it closes once the answer is read. ctx bounds the request. It returns the error +// of a uri that does not form a URL, of a request that never got an answer, and of an answer whose +// body could not be read in full or closed. +func (dc *DockerCompose) VisitWebEndpoint(ctx context.Context, host, uri string) (*WebEndpointResponse, error) { + return visit(ctx, &http.Transport{DisableKeepAlives: true}, dc.BaseURL()+uri, host) +} + +// VisitWebEndpointOverTLS sends GET uri to the gateway's HTTPS port the way +// [DockerCompose.VisitWebEndpoint] does to its HTTP port, naming host in TLS's server name +// indication as well as in Host. config verifies the gateway's certificate; its ServerName is +// replaced with host. It returns the errors [DockerCompose.VisitWebEndpoint] does, among them a +// certificate config refuses. +func (dc *DockerCompose) VisitWebEndpointOverTLS(ctx context.Context, config *tls.Config, host, uri string) (*WebEndpointResponse, error) { + config = config.Clone() + config.ServerName = host + + return visit(ctx, &http.Transport{DisableKeepAlives: true, TLSClientConfig: config}, "https://"+dc.HTTPSAddress()+uri, host) +} + +func visit(ctx context.Context, transport *http.Transport, url, host string) (*WebEndpointResponse, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Host = host + + resp, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + return nil, err + } + + body, err := io.ReadAll(resp.Body) + if err := errors.Join(err, resp.Body.Close()); err != nil { + return nil, err + } + + return &WebEndpointResponse{StatusCode: resp.StatusCode, Body: string(body)}, nil +} diff --git a/tests/ssh_test.go b/tests/ssh_test.go index 3301e2312d0..5c74cc9272c 100644 --- a/tests/ssh_test.go +++ b/tests/ssh_test.go @@ -57,6 +57,12 @@ func NewAgentContainerWithConnectionVersion(version int) NewAgentContainerOption } } +func NewAgentContainerWithTenant(tenant string) NewAgentContainerOption { + return func(envs map[string]string) { + envs["SHELLHUB_TENANT_ID"] = tenant + } +} + func NewAgentContainerWithHostname(hostname string) NewAgentContainerOption { return func(envs map[string]string) { envs["SHELLHUB_PREFERRED_HOSTNAME"] = hostname diff --git a/tests/web_endpoints_test.go b/tests/web_endpoints_test.go new file mode 100644 index 00000000000..1231c67d24c --- /dev/null +++ b/tests/web_endpoints_test.go @@ -0,0 +1,589 @@ +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "io" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/shellhub-io/shellhub/pkg/clock" + "github.com/shellhub-io/shellhub/pkg/models" + "github.com/shellhub-io/shellhub/pkg/testimage" + "github.com/shellhub-io/shellhub/tests/environment" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + deviceServiceHost = "127.0.0.1" + intruderUsername = "intruder" + intruderNamespaceName = "intruderspace" + intruderNamespace = "00000000-0000-4000-0000-000000000022" + deviceTLSDomain = "device.example.com" + webEndpointDomain = "localhost" + unknownAddress = "0123456789abcdef0123456789abcdef" + webEndpointNotFound = "web endpoint not found" + gatewayRootCertificate = "/data/pki/authorities/local/root.crt" + shortTTL = 3 +) + +var ( + webEndpointAddress = regexp.MustCompile(`^[0-9a-f]{32}$`) + unknownDeviceUID = strings.Repeat("0", 64) +) + +type deviceService struct { + name string + httpPort int + tlsPort int +} + +func (s deviceService) plain(uid string, ttl int) environment.WebEndpointCreate { + return environment.WebEndpointCreate{UID: uid, Host: deviceServiceHost, Port: s.httpPort, TTL: ttl} +} + +func (s deviceService) secure(uid string, verify bool) environment.WebEndpointCreate { + return environment.WebEndpointCreate{ + UID: uid, + Host: deviceServiceHost, + Port: s.tlsPort, + TTL: -1, + TLS: &environment.WebEndpointTLS{Enabled: true, Verify: verify, Domain: deviceTLSDomain}, + } +} + +type deviceSeen struct { + Name string `json:"name"` + Host string `json:"host"` + URI string `json:"uri"` + TLS bool `json:"tls"` +} + +// TestEnterpriseWebEndpoints covers web endpoints on an enterprise instance that enables them: what +// the API refuses to create, the address and lifetime of what it creates, the gateway carrying a +// request sent to an endpoint's address through the agent into an HTTP or TLS service on the +// device, the deprecated tunnel API over the same endpoints, and a device merge handing a device's +// endpoints to the device it merged into. The device's services listen in the agents' network +// namespace and answer with the name they were started with and what they received. +func TestEnterpriseWebEndpoints(t *testing.T) { + ctx := context.Background() + + compose := newEnterpriseEnvironment(t, ctx, environment.New(t, run).WithEnv("SHELLHUB_WEB_ENDPOINTS", "true")) + _, device := startAcceptedAgent(t, ctx, compose) + + first := startDeviceService(t, compose, deviceService{name: "first", httpPort: 8001, tlsPort: 8443}) + second := startDeviceService(t, compose, deviceService{name: "second", httpPort: 8002, tlsPort: 8444}) + + t.Run("creating", func(t *testing.T) { testWebEndpointCreation(t, compose, device, first.plain(device.UID, -1)) }) + t.Run("proxying", func(t *testing.T) { testWebEndpointProxy(t, compose, device, first, second) }) + t.Run("lifetime", func(t *testing.T) { + testWebEndpointLifetime(t, compose, first.plain(device.UID, -1), second.plain(device.UID, -1)) + }) + t.Run("deprecated tunnel API", func(t *testing.T) { testTunnelAPI(t, compose, device, second) }) + t.Run("another namespace", func(t *testing.T) { testWebEndpointAnotherNamespace(t, ctx, compose, device, first) }) + t.Run("device merge", func(t *testing.T) { testWebEndpointDeviceMerge(t, ctx, compose, first) }) +} + +// TestEnterpriseWebEndpointsDisabled covers an enterprise instance that leaves web endpoints off, +// as it does by default: neither the API nor the gateway serves them. +func TestEnterpriseWebEndpointsDisabled(t *testing.T) { + compose := newEnterpriseEnvironment(t, context.Background(), environment.New(t, run)) + + t.Run("the API serves no web endpoint route", func(t *testing.T) { + requests := []struct { + method string + path string + }{ + {method: http.MethodGet, path: "/api/web-endpoints"}, + {method: http.MethodPost, path: "/api/web-endpoints"}, + {method: http.MethodDelete, path: "/api/web-endpoints/" + unknownAddress}, + {method: http.MethodGet, path: "/api/devices/" + unknownDeviceUID + "/tunnels"}, + {method: http.MethodPost, path: "/api/devices/" + unknownDeviceUID + "/tunnels"}, + } + + for _, req := range requests { + resp, err := compose.R(t.Context()).SetBody(map[string]any{}).Execute(req.method, req.path) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode(), "%s %s: %s", req.method, req.path, resp.String()) + } + }) + + t.Run("the gateway serves no site for an endpoint's address", func(t *testing.T) { + resp := visitWebEndpoint(t, compose, unknownAddress+"."+webEndpointDomain, "/") + + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + assert.Empty(t, resp.Body) + }) +} + +// TestEnterpriseWebEndpointsTLS covers the gateway serving web endpoints over HTTPS on a domain no +// public authority signs for: it presents a certificate its own authority issued for the address, +// and carries the request to the server. +func TestEnterpriseWebEndpointsTLS(t *testing.T) { + compose := environment.New(t, run). + WithEdition(environment.EditionEnterprise). + WithEnv("SHELLHUB_WEB_ENDPOINTS", "true"). + WithAutoSSL(). + Up(context.Background()) + t.Cleanup(compose.Down) + + roots := x509.NewCertPool() + roots.AddCert(gatewayAuthority(t, compose)) + + var resp *environment.WebEndpointResponse + + require.EventuallyWithT(t, func(tt *assert.CollectT) { + var err error + + resp, err = compose.VisitWebEndpointOverTLS(t.Context(), &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, unknownAddress+"."+webEndpointDomain, "/") + assert.NoError(tt, err) + }, 30*time.Second, time.Second) + + assert.Equal(t, http.StatusForbidden, resp.StatusCode) + assert.Contains(t, resp.Body, webEndpointNotFound) +} + +func testWebEndpointCreation(t *testing.T, compose *environment.DockerCompose, device *models.Device, valid environment.WebEndpointCreate) { + t.Helper() + + refusals := []struct { + description string + change func(req *environment.WebEndpointCreate) + status int + }{ + {description: "a host that is not an IP address", change: func(req *environment.WebEndpointCreate) { req.Host = "localhost" }, status: http.StatusBadRequest}, + {description: "port 0", change: func(req *environment.WebEndpointCreate) { req.Port = 0 }, status: http.StatusBadRequest}, + {description: "a TTL of 0", change: func(req *environment.WebEndpointCreate) { req.TTL = 0 }, status: http.StatusBadRequest}, + {description: "a TTL below -1", change: func(req *environment.WebEndpointCreate) { req.TTL = -2 }, status: http.StatusBadRequest}, + {description: "a device the namespace does not hold", change: func(req *environment.WebEndpointCreate) { req.UID = unknownDeviceUID }, status: http.StatusNotFound}, + {description: "TLS without a domain", change: func(req *environment.WebEndpointCreate) { req.TLS = &environment.WebEndpointTLS{Enabled: true} }, status: http.StatusBadRequest}, + } + + for _, tc := range refusals { + t.Run("an endpoint for "+tc.description+" is refused", func(t *testing.T) { + req := valid + tc.change(&req) + + requireCreateRefused(t, compose, req, tc.status) + }) + } + + t.Run("an endpoint for a device not yet accepted is refused", func(t *testing.T) { + pending := enrollPendingDevice(t, compose, "web-endpoint-pending", "02:00:00:00:22:01") + + req := valid + req.UID = pending.UID + + requireCreateRefused(t, compose, req, http.StatusForbidden) + }) + + t.Run("an endpoint for a namespace that no longer exists is refused", func(t *testing.T) { + const username, password = "orphan", "password" + + compose.NewUser(t, username, "orphan@shellhub.example.com", password) + compose.NewNamespace(t, username, "orphanspace", "00000000-0000-4000-0000-000000002201", "") + token := compose.AuthUser(t, username, password).Token + + compose.DeleteNamespace(t, "orphanspace") + + resp, err := compose.Anonymous(t.Context()).SetAuthToken(token).SetBody(valid).Post("/api/web-endpoints") + require.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode(), resp.String()) + }) + + t.Run("an endpoint answers under a 32 hex digit address of the web endpoint domain", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, valid) + + assert.Regexp(t, webEndpointAddress, endpoint.Address) + assert.Equal(t, endpoint.Address+"."+webEndpointDomain, endpoint.FullAddress) + assert.Equal(t, ShellHubNamespace, endpoint.Namespace) + assert.Equal(t, device.UID, endpoint.DeviceUID) + assert.Equal(t, -1, endpoint.TTL) + assert.True(t, endpoint.ExpiresIn.IsZero(), "an endpoint with a TTL of -1 has no expiry") + }) + + t.Run("the same device, host and port keep their endpoint and get a new address once it is deleted", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, valid) + + assert.Equal(t, endpoint.Address, compose.CreateWebEndpoint(t, valid).Address) + + resp, err := compose.DeleteWebEndpoint(t.Context(), endpoint.Address) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + recreated := compose.CreateWebEndpoint(t, valid) + assert.Regexp(t, webEndpointAddress, recreated.Address) + assert.NotEqual(t, endpoint.Address, recreated.Address) + }) +} + +func testWebEndpointProxy(t *testing.T, compose *environment.DockerCompose, device *models.Device, first, second deviceService) { + t.Helper() + + t.Run("a request to an endpoint's address reaches the device's service with its path and the address as Host", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, first.plain(device.UID, -1)) + + seen := requireDeviceAnswer(t, compose, endpoint.FullAddress, "/some/path?query=value") + assert.Equal(t, first.name, seen.Name) + assert.Equal(t, "/some/path?query=value", seen.URI) + assert.Equal(t, endpoint.FullAddress, seen.Host) + assert.False(t, seen.TLS) + }) + + t.Run("requests to two endpoints each reach their own service", func(t *testing.T) { + one := compose.CreateWebEndpoint(t, first.plain(device.UID, -1)) + two := compose.CreateWebEndpoint(t, second.plain(device.UID, -1)) + + for range 3 { + assert.Equal(t, first.name, requireDeviceAnswer(t, compose, one.FullAddress, "/").Name) + assert.Equal(t, second.name, requireDeviceAnswer(t, compose, two.FullAddress, "/").Name) + } + }) + + t.Run("a request to a deleted endpoint's address is refused", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, first.plain(device.UID, -1)) + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + + resp, err := compose.DeleteWebEndpoint(t.Context(), endpoint.Address) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + requireEndpointRefused(t, compose, endpoint.FullAddress) + }) + + t.Run("a request to an address no endpoint has is refused", func(t *testing.T) { + requireEndpointRefused(t, compose, unknownAddress+"."+webEndpointDomain) + }) + + t.Run("an endpoint over TLS that does not verify reaches a self-signed service, naming its domain as Host", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, first.secure(device.UID, false)) + + seen := requireDeviceAnswer(t, compose, endpoint.FullAddress, "/secure?query=value") + assert.Equal(t, first.name, seen.Name) + assert.True(t, seen.TLS) + assert.Equal(t, "/secure?query=value", seen.URI) + assert.Equal(t, deviceTLSDomain, seen.Host) + }) + + t.Run("an endpoint over TLS that verifies refuses a self-signed service", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, first.secure(device.UID, true)) + + resp := visitWebEndpoint(t, compose, endpoint.FullAddress, "/") + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) + assert.Contains(t, resp.Body, "failed to connect to the port on device") + }) + + t.Run("a subdomain that is not an address is not served", func(t *testing.T) { + for _, path := range []string{"/api/info", "/internal/metrics"} { + resp := visitWebEndpoint(t, compose, "console."+webEndpointDomain, path) + + assert.Equal(t, http.StatusNotFound, resp.StatusCode, "%s: %s", path, resp.Body) + } + }) + + t.Run("a request to the instance's own domain still reaches the API", func(t *testing.T) { + resp := visitWebEndpoint(t, compose, webEndpointDomain, "/api/info") + + assert.Equal(t, http.StatusOK, resp.StatusCode, resp.Body) + }) +} + +func testWebEndpointLifetime(t *testing.T, compose *environment.DockerCompose, endless, elsewhere environment.WebEndpointCreate) { + t.Helper() + + withTTL := func(ttl int) environment.WebEndpointCreate { + req := endless + req.TTL = ttl + + return req + } + + t.Run("an endpoint never accessed expires its TTL after its creation", func(t *testing.T) { + const ttl = 3600 + + endpoint := listedWebEndpoint(t, compose, compose.CreateWebEndpoint(t, withTTL(ttl)).Address) + + assert.WithinDuration(t, endpoint.CreatedAt.Add(ttl*time.Second), endpoint.ExpiresIn, time.Millisecond) + }) + + t.Run("an accessed endpoint expires its TTL after its last access", func(t *testing.T) { + const ttl = 3600 + + endpoint := compose.CreateWebEndpoint(t, withTTL(ttl)) + created := listedWebEndpoint(t, compose, endpoint.Address) + + time.Sleep(2 * time.Second) + + before := clock.Now() + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + after := clock.Now() + + accessed := listedWebEndpoint(t, compose, endpoint.Address) + assert.True(t, accessed.ExpiresIn.After(created.ExpiresIn.Add(time.Second)), "the expiry %s did not move past %s", accessed.ExpiresIn, created.ExpiresIn) + assert.WithinRange(t, accessed.ExpiresIn.Add(-ttl*time.Second), before.Add(-time.Second), after.Add(time.Second)) + }) + + t.Run("an endpoint answers until its TTL runs out and is refused after it", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, withTTL(shortTTL)) + + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + requireRefusedOnceIdleFor(t, compose, endpoint.FullAddress, shortTTL) + }) + + t.Run("an endpoint accessed within its TTL keeps answering past it", func(t *testing.T) { + const ttl = 5 + + endpoint := compose.CreateWebEndpoint(t, withTTL(ttl)) + pastFirstExpiry := endpoint.CreatedAt.Add(ttl*time.Second + time.Second) + + for clock.Now().Before(pastFirstExpiry) { + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + time.Sleep(time.Second) + } + + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + requireRefusedOnceIdleFor(t, compose, endpoint.FullAddress, ttl) + }) + + t.Run("an endpoint with a TTL of -1 keeps answering after one with a TTL has expired", func(t *testing.T) { + finite := compose.CreateWebEndpoint(t, withTTL(shortTTL)) + infinite := compose.CreateWebEndpoint(t, elsewhere) + + requireRefusedOnceIdleFor(t, compose, finite.FullAddress, shortTTL) + + requireDeviceAnswer(t, compose, infinite.FullAddress, "/") + assert.True(t, listedWebEndpoint(t, compose, infinite.Address).ExpiresIn.IsZero()) + }) + + t.Run("an expired endpoint can be deleted", func(t *testing.T) { + endpoint := compose.CreateWebEndpoint(t, withTTL(shortTTL)) + requireRefusedOnceIdleFor(t, compose, endpoint.FullAddress, shortTTL) + + resp, err := compose.DeleteWebEndpoint(t.Context(), endpoint.Address) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + for _, listed := range compose.ListWebEndpoints(t) { + assert.NotEqual(t, endpoint.Address, listed.Address) + } + }) +} + +func testTunnelAPI(t *testing.T, compose *environment.DockerCompose, device *models.Device, service deviceService) { + t.Helper() + + resp, err := compose.PostTunnel(t.Context(), device.UID, environment.TunnelCreate{Host: deviceServiceHost, Port: service.httpPort, TTL: -1}) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + tunnels := compose.ListTunnels(t, device.UID) + require.Len(t, tunnels, 1) + + tunnel := tunnels[0] + t.Cleanup(func() { _, _ = compose.DeleteTunnel(context.WithoutCancel(t.Context()), device.UID, tunnel.Address) }) + + assert.Regexp(t, webEndpointAddress, tunnel.Address) + assert.Equal(t, tunnel.Address+"."+webEndpointDomain, tunnel.FullAddress) + assert.Equal(t, device.UID, tunnel.Device) + assert.Equal(t, deviceServiceHost, tunnel.Host) + assert.Equal(t, service.httpPort, tunnel.Port) + assert.Equal(t, -1, tunnel.TTL) + + assert.Equal(t, service.name, requireDeviceAnswer(t, compose, tunnel.FullAddress, "/").Name) + + resp, err = compose.DeleteTunnel(t.Context(), device.UID, tunnel.Address) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) + + assert.Empty(t, compose.ListTunnels(t, device.UID)) + requireEndpointRefused(t, compose, tunnel.FullAddress) +} + +func testWebEndpointAnotherNamespace(t *testing.T, ctx context.Context, compose *environment.DockerCompose, device *models.Device, service deviceService) { + t.Helper() + + endpoint := compose.CreateWebEndpoint(t, service.plain(device.UID, -1)) + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + + owner := compose.AuthUser(t, ShellHubUsername, ShellHubPassword) + + compose.NewUser(t, intruderUsername, "intruder@ossystems.com.br", ShellHubPassword) + compose.NewNamespace(t, intruderUsername, intruderNamespaceName, intruderNamespace, "") + compose.JWT(compose.AuthUser(t, intruderUsername, ShellHubPassword).Token) + t.Cleanup(func() { compose.JWT(owner.Token) }) + + _, foreign := startAcceptedAgent(t, ctx, compose, + NewAgentContainerWithTenant(intruderNamespace), + NewAgentContainerWithIdentity("020000002203"), + NewAgentContainerWithHostname("intruding"), + ) + + resp, err := compose.DeleteTunnel(t.Context(), foreign.UID, endpoint.Address) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode(), "deleting through a device of its own: %s", resp.String()) + + resp, err = compose.DeleteWebEndpoint(t.Context(), endpoint.Address) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode(), "deleting by address: %s", resp.String()) + + assert.Equal(t, service.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) +} + +func testWebEndpointDeviceMerge(t *testing.T, ctx context.Context, compose *environment.DockerCompose, service deviceService) { + t.Helper() + + same := []NewAgentContainerOption{NewAgentContainerWithIdentity("020000002202"), NewAgentContainerWithHostname("merging")} + + oldAgent, old := startAcceptedAgent(t, ctx, compose, same...) + + endpoint := compose.CreateWebEndpoint(t, service.plain(old.UID, -1)) + requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") + + require.NoError(t, oldAgent.Stop(ctx, nil)) + compose.AwaitDeviceOffline(t, old.UID) + + _, merged := startAcceptedAgent(t, ctx, compose, same...) + require.NotEqual(t, old.UID, merged.UID) + + assert.Equal(t, merged.UID, listedWebEndpoint(t, compose, endpoint.Address).DeviceUID) + assert.Equal(t, service.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) +} + +func startDeviceService(t *testing.T, compose *environment.DockerCompose, service deviceService) deviceService { + t.Helper() + + gateway := compose.Service(environment.ServiceGateway).GetContainerID() + + buildArgs, err := testimage.BuildArgs("..") + require.NoError(t, err) + + c, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Repo: "device-http", + Tag: "test", + Context: "cmd/device-http", + KeepImage: true, + BuildArgs: buildArgs, + }, + Env: map[string]string{ + "NAME": service.name, + "HTTP_ADDRESS": deviceServiceHost + ":" + strconv.Itoa(service.httpPort), + "TLS_ADDRESS": deviceServiceHost + ":" + strconv.Itoa(service.tlsPort), + "TLS_DOMAIN": deviceTLSDomain, + }, + Labels: run.Labels(), + HostConfigModifier: func(hc *container.HostConfig) { + hc.NetworkMode = container.NetworkMode("container:" + gateway) + }, + WaitingFor: wait.ForLog("listening"), + }, + Started: true, + Logger: log.New(io.Discard, "", log.LstdFlags), + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = c.Terminate(context.WithoutCancel(t.Context())) }) + + return service +} + +func gatewayAuthority(t *testing.T, compose *environment.DockerCompose) *x509.Certificate { + t.Helper() + + var data []byte + + require.EventuallyWithT(t, func(tt *assert.CollectT) { + reader, err := compose.Service(environment.ServiceGateway).CopyFileFromContainer(t.Context(), gatewayRootCertificate) + if !assert.NoError(tt, err) { + return + } + + defer reader.Close() //nolint:errcheck // a failed read is retried; the close changes nothing + + data, err = io.ReadAll(reader) + assert.NoError(tt, err) + }, 30*time.Second, time.Second) + + block, _ := pem.Decode(data) + require.NotNil(t, block, "the gateway's root certificate is not PEM") + + certificate, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + return certificate +} + +func listedWebEndpoint(t *testing.T, compose *environment.DockerCompose, address string) environment.WebEndpoint { + t.Helper() + + for _, endpoint := range compose.ListWebEndpoints(t) { + if endpoint.Address == address { + return endpoint + } + } + + require.Failf(t, "the web endpoint is not listed", "address %s", address) + + return environment.WebEndpoint{} +} + +func requireCreateRefused(t *testing.T, compose *environment.DockerCompose, req environment.WebEndpointCreate, status int) { + t.Helper() + + listed := len(compose.ListWebEndpoints(t)) + + resp, err := compose.PostWebEndpoint(t.Context(), req, nil) + require.NoError(t, err) + require.Equal(t, status, resp.StatusCode(), resp.String()) + assert.Len(t, compose.ListWebEndpoints(t), listed, "a refused create left an endpoint behind") +} + +func visitWebEndpoint(t *testing.T, compose *environment.DockerCompose, host, uri string) *environment.WebEndpointResponse { + t.Helper() + + resp, err := compose.VisitWebEndpoint(t.Context(), host, uri) + require.NoError(t, err) + + return resp +} + +func requireDeviceAnswer(t *testing.T, compose *environment.DockerCompose, host, uri string) deviceSeen { + t.Helper() + + resp := visitWebEndpoint(t, compose, host, uri) + require.Equal(t, http.StatusOK, resp.StatusCode, resp.Body) + + seen := deviceSeen{} + require.NoError(t, json.Unmarshal([]byte(resp.Body), &seen), resp.Body) + + return seen +} + +func requireEndpointRefused(t *testing.T, compose *environment.DockerCompose, host string) { + t.Helper() + + resp := visitWebEndpoint(t, compose, host, "/") + require.Equal(t, http.StatusForbidden, resp.StatusCode, resp.Body) + assert.Contains(t, resp.Body, webEndpointNotFound) +} + +func requireRefusedOnceIdleFor(t *testing.T, compose *environment.DockerCompose, host string, ttl int) { + t.Helper() + + time.Sleep(time.Duration(ttl)*time.Second + time.Second) + + requireEndpointRefused(t, compose, host) +} diff --git a/ui/apps/console/e2e/billing.spec.ts b/ui/apps/console/e2e/billing.spec.ts index 04c872d2b88..deecfc66894 100644 --- a/ui/apps/console/e2e/billing.spec.ts +++ b/ui/apps/console/e2e/billing.spec.ts @@ -189,8 +189,13 @@ async function submitNamespaceDeletion(page: Page, owner: Owner) { await dialog .getByLabel(`Type "${owner.namespace}" to confirm`) .fill(owner.namespace); + const answered = page.waitForResponse( + (r) => + r.request().method() === "DELETE" && + r.url().endsWith(`/api/namespaces/${owner.tenant}`), + ); await dialog.getByRole("button", { name: "Delete namespace" }).click(); - return dialog; + return { dialog, deletion: await answered }; } test.describe("Billing", () => { @@ -283,7 +288,8 @@ test.describe("Billing", () => { owner.customer = subscription.customer; await page.goto("/settings/general"); - const dialog = await submitNamespaceDeletion(page, owner); + const { dialog, deletion } = await submitNamespaceDeletion(page, owner); + expect(deletion.status()).toBe(402); await expect( dialog.getByText("Couldn't delete the namespace."), @@ -295,8 +301,9 @@ test.describe("Billing", () => { .poll(() => readSubscriptionStatus(owner), { timeout: 30_000 }) .toBe("canceled"); await page.reload(); - await submitNamespaceDeletion(page, owner); + const { deletion: retried } = await submitNamespaceDeletion(page, owner); + expect(retried.status()).toBe(200); - await expect.poll(() => readTenants(owner)).toEqual([]); + expect(await readTenants(owner)).toEqual([]); }); });