diff --git a/Makefile b/Makefile index 39a1f7ae..73cf9284 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ KIND_NAME ?= apisix-ingress-cluster KIND_NODE_IMAGE ?= kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f DASHBOARD_VERSION ?= dev -ADC_VERSION ?= 0.27.1 +ADC_VERSION ?= 0.29.0 DIR := $(shell pwd) diff --git a/api/adc/types.go b/api/adc/types.go index 4ff25a71..a18274bc 100644 --- a/api/adc/types.go +++ b/api/adc/types.go @@ -807,6 +807,11 @@ type Config struct { TlsVerify bool BackendType string + // CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + // control plane, in place of the system trust store. Only meaningful when + // TlsVerify is true. + CaBundle string + // BypassCache makes the ADC server drop the in-memory baseline it holds for this // cacheKey and re-derive it from the data plane before computing the diff. It is a // per-request flag set on the sync path, not part of the translated configuration. @@ -820,10 +825,12 @@ func (c Config) MarshalJSON() ([]byte, error) { Name string `json:"name"` ServerAddrs []string `json:"serverAddrs"` TlsVerify bool `json:"tlsVerify"` + HasCaBundle bool `json:"hasCaBundle"` }{ Name: c.Name, ServerAddrs: c.ServerAddrs, TlsVerify: c.TlsVerify, + HasCaBundle: c.CaBundle != "", }) } diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go index 680fa8a9..629c780c 100644 --- a/api/v1alpha1/gatewayproxy_types.go +++ b/api/v1alpha1/gatewayproxy_types.go @@ -120,6 +120,7 @@ type ControlPlaneAuth struct { // ControlPlaneProvider defines configuration for control plane provider. // +kubebuilder:validation:XValidation:rule="has(self.endpoints) != has(self.service)" // +kubebuilder:validation:XValidation:rule="oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode",message="mode is immutable" +// +kubebuilder:validation:XValidation:rule="!has(self.caBundle) || self.caBundle.contains('-----BEGIN CERTIFICATE-----')",message="caBundle must be a PEM-encoded certificate" type ControlPlaneProvider struct { // Mode specifies the mode of control plane provider. // Can be `apisix` or `apisix-standalone`. @@ -136,6 +137,13 @@ type ControlPlaneProvider struct { // +optional TlsVerify *bool `json:"tlsVerify,omitempty"` + // CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + // control plane's TLS certificate, in place of the system trust store. + // Set it when the control plane uses a self-signed or private CA certificate. + // It has no effect when tlsVerify is false. + // +optional + CaBundle string `json:"caBundle,omitempty"` + // Auth specifies the authentication configuration. // +kubebuilder:validation:Required Auth ControlPlaneAuth `json:"auth"` diff --git a/config/crd-nocel/apisix.apache.org_v2.yaml b/config/crd-nocel/apisix.apache.org_v2.yaml index 9e4cc186..bde3da1c 100644 --- a/config/crd-nocel/apisix.apache.org_v2.yaml +++ b/config/crd-nocel/apisix.apache.org_v2.yaml @@ -2273,6 +2273,13 @@ spec: required: - type type: object + caBundle: + description: |- + CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + control plane's TLS certificate, in place of the system trust store. + Set it when the control plane uses a self-signed or private CA certificate. + It has no effect when tlsVerify is false. + type: string endpoints: description: Endpoints specifies the list of control plane endpoints. diff --git a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml index 23a7ed50..617226d5 100644 --- a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml +++ b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml @@ -120,6 +120,13 @@ spec: - message: adminKey must be specified when type is AdminKey rule: 'self.type == ''AdminKey'' ? has(self.adminKey) : true' + caBundle: + description: |- + CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + control plane's TLS certificate, in place of the system trust store. + Set it when the control plane uses a self-signed or private CA certificate. + It has no effect when tlsVerify is false. + type: string endpoints: description: Endpoints specifies the list of control plane endpoints. @@ -158,6 +165,9 @@ spec: - message: mode is immutable rule: oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode + - message: caBundle must be a PEM-encoded certificate + rule: '!has(self.caBundle) || self.caBundle.contains(''-----BEGIN + CERTIFICATE-----'')' type: description: Type specifies the type of provider. Can only be `ControlPlane`. diff --git a/docs/en/latest/reference/api-reference.md b/docs/en/latest/reference/api-reference.md index 8def827e..57684a4b 100644 --- a/docs/en/latest/reference/api-reference.md +++ b/docs/en/latest/reference/api-reference.md @@ -313,6 +313,7 @@ ControlPlaneProvider defines configuration for control plane provider. | `endpoints` _string array_ | Endpoints specifies the list of control plane endpoints. | | `service` _[ProviderService](#providerservice)_ | | | `tlsVerify` _boolean_ | TlsVerify specifies whether to verify the TLS certificate of the control plane. | +| `caBundle` _string_ | CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the control plane's TLS certificate, in place of the system trust store. Set it when the control plane uses a self-signed or private CA certificate. It has no effect when tlsVerify is false. | | `auth` _[ControlPlaneAuth](#controlplaneauth)_ | Auth specifies the authentication configuration. | diff --git a/internal/adc/client/executor.go b/internal/adc/client/executor.go index 2fe32009..45b42806 100644 --- a/internal/adc/client/executor.go +++ b/internal/adc/client/executor.go @@ -84,7 +84,11 @@ type ADCServerOpts struct { LabelSelector map[string]string `json:"labelSelector,omitempty"` IncludeResourceType []string `json:"includeResourceType,omitempty"` TlsSkipVerify *bool `json:"tlsSkipVerify,omitempty"` - CacheKey string `json:"cacheKey"` + // CaCert is the PEM-encoded CA certificate (or bundle) the ADC server verifies + // the control plane against. Older ADC servers ignore it, and omitempty keeps + // requests without a CA bundle byte for byte what they were. + CaCert string `json:"caCert,omitempty"` + CacheKey string `json:"cacheKey"` // BypassCache is only accepted by the /sync task of ADC >= 0.27.0. Both ADC task // schemas reject unknown fields, so omitempty is what keeps every other request -- // /validate, and every sync that is not recovering from a rejection -- byte for byte @@ -103,6 +107,7 @@ func (r ADCServerRequest) MarshalLog() any { "labelSelector": r.Task.Opts.LabelSelector, "includeResourceType": r.Task.Opts.IncludeResourceType, "tlsSkipVerify": r.Task.Opts.TlsSkipVerify, + "hasCaCert": r.Task.Opts.CaCert != "", "cacheKey": r.Task.Opts.CacheKey, "config": r.Task.Config.MarshalLog(), } @@ -244,7 +249,7 @@ func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, server } // Build HTTP request - req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, pathSync) + req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, pathSync) if err != nil { return fmt.Errorf("failed to build HTTP request: %w", err) } @@ -278,7 +283,7 @@ func (e *HTTPADCExecutor) runHTTPValidateForSingleServer(ctx context.Context, se return fmt.Errorf("failed to load resources from file %s: %w", filePath, err) } - req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, pathValidate) + req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, pathValidate) if err != nil { return fmt.Errorf("failed to build validate request: %w", err) } @@ -349,7 +354,7 @@ func (e *HTTPADCExecutor) loadResourcesFromFile(filePath string) (*adctypes.Reso } // buildHTTPRequest builds the HTTP request for ADC Server -func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, method string, path string) (*http.Request, error) { +func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, path string) (*http.Request, error) { // Prepare request body tlsVerify := config.TlsVerify bypassCache := path == pathSync && config.BypassCache @@ -362,6 +367,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin LabelSelector: labels, IncludeResourceType: types, TlsSkipVerify: ptr.To(!tlsVerify), + CaCert: config.CaBundle, CacheKey: config.Name, BypassCache: bypassCache, }, @@ -385,10 +391,11 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin "labelSelector", labels, "includeResourceType", types, "tlsSkipVerify", !tlsVerify, + "hasCaCert", config.CaBundle != "", ) // Create HTTP request - req, err := http.NewRequestWithContext(ctx, method, e.serverURL+path, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, e.serverURL+path, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create HTTP request: %w", err) } diff --git a/internal/adc/client/executor_test.go b/internal/adc/client/executor_test.go index 9e7ee71c..f488bb6d 100644 --- a/internal/adc/client/executor_test.go +++ b/internal/adc/client/executor_test.go @@ -22,7 +22,6 @@ import ( "encoding/json" "errors" "io" - "net/http" "testing" "github.com/go-logr/logr" @@ -41,7 +40,7 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) { build := func(config adctypes.Config, path string) (ADCServerOpts, string) { req, err := e.buildHTTPRequest(context.Background(), "http://apisix:9180", config, nil, nil, - &adctypes.Resources{}, http.MethodPut, path) + &adctypes.Resources{}, path) require.NoError(t, err) body, err := io.ReadAll(req.Body) require.NoError(t, err) @@ -70,6 +69,37 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) { assert.NotContains(t, raw, "bypassCache") } +func TestHTTPADCExecutorBuildHTTPRequestCaCert(t *testing.T) { + e := &HTTPADCExecutor{ + serverURL: "http://127.0.0.1:3000", + log: logr.Discard(), + } + + build := func(config adctypes.Config) (ADCServerOpts, string) { + req, err := e.buildHTTPRequest(context.Background(), "https://apisix:9180", config, nil, nil, + &adctypes.Resources{}, pathSync) + require.NoError(t, err) + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + var parsed ADCServerRequest + require.NoError(t, json.Unmarshal(body, &parsed)) + return parsed.Task.Opts, string(body) + } + + // Without a CA bundle the request stays what an ADC server that predates caCert + // already accepts. + opts, raw := build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true}) + assert.Empty(t, opts.CaCert) + assert.NotContains(t, raw, "caCert") + + const caCert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----" + opts, raw = build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true, CaBundle: caCert}) + assert.Equal(t, caCert, opts.CaCert) + assert.Contains(t, raw, "caCert") + // verification stays on, otherwise the bundle would be pointless + assert.Equal(t, false, *opts.TlsSkipVerify) +} + // confVersionError is what a push carrying a conf_version older than the data plane's // comes back as, once the ADC server has relayed the rejection to us. func confVersionError() error { diff --git a/internal/adc/translator/gatewayproxy.go b/internal/adc/translator/gatewayproxy.go index 3a9a00f4..adf958a4 100644 --- a/internal/adc/translator/gatewayproxy.go +++ b/internal/adc/translator/gatewayproxy.go @@ -18,6 +18,8 @@ package translator import ( + "crypto/x509" + "encoding/pem" "fmt" "net" "strconv" @@ -56,6 +58,17 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte cfg.TlsVerify = *cp.TlsVerify } + if cp.CaBundle != "" { + // reject unusable CA material here rather than at connect time + if err := validateCaBundle(cp.CaBundle); err != nil { + return nil, err + } + if !cfg.TlsVerify { + t.Log.Info("caBundle is ignored because tlsVerify is disabled", "gatewayproxy", utils.NamespacedNameKind(gatewayProxy)) + } + cfg.CaBundle = cp.CaBundle + } + if cp.Auth.Type == v1alpha1.AuthTypeAdminKey && cp.Auth.AdminKey != nil { if cp.Auth.AdminKey.ValueFrom != nil && cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef @@ -142,3 +155,28 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte return &cfg, nil } + +// validateCaBundle parses every certificate in the bundle. x509.CertPool skips +// blocks it cannot decode, so a bundle whose second certificate is broken would +// otherwise reach the ADC server and fail there instead. +func validateCaBundle(caBundle string) error { + var count int + for rest := []byte(caBundle); len(rest) > 0; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + return fmt.Errorf("invalid caBundle: expected a CERTIFICATE block, got %s", block.Type) + } + if _, err := x509.ParseCertificate(block.Bytes); err != nil { + return fmt.Errorf("invalid caBundle: %w", err) + } + count++ + } + if count == 0 { + return errors.New("invalid caBundle: no PEM-encoded certificate found") + } + return nil +} diff --git a/internal/adc/translator/gatewayproxy_test.go b/internal/adc/translator/gatewayproxy_test.go new file mode 100644 index 00000000..3429b711 --- /dev/null +++ b/internal/adc/translator/gatewayproxy_test.go @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/apache/apisix-ingress-controller/api/v1alpha1" + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +func newGatewayProxy(tlsVerify *bool, caBundle string) *v1alpha1.GatewayProxy { + return &v1alpha1.GatewayProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gp", + }, + Spec: v1alpha1.GatewayProxySpec{ + Provider: &v1alpha1.GatewayProxyProvider{ + Type: v1alpha1.ProviderTypeControlPlane, + ControlPlane: &v1alpha1.ControlPlaneProvider{ + Endpoints: []string{"https://cp.example.com:9180"}, + TlsVerify: tlsVerify, + CaBundle: caBundle, + Auth: v1alpha1.ControlPlaneAuth{ + Type: v1alpha1.AuthTypeAdminKey, + AdminKey: &v1alpha1.AdminKeyAuth{ + Value: "admin-key", + }, + }, + }, + }, + }, + } +} + +func TestTranslateGatewayProxyToConfigCaBundle(t *testing.T) { + t.Run("carries the CA bundle into the config", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), testCACert), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.True(t, cfg.TlsVerify) + assert.Equal(t, testCACert, cfg.CaBundle) + }) + + t.Run("leaves the CA bundle empty when unset", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), ""), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Empty(t, cfg.CaBundle) + }) + + // every certificate is parsed: x509.CertPool silently skips the blocks it + // cannot decode, which would let a broken one through to the ADC server. + for name, caBundle := range map[string]string{ + "not PEM at all": "not-a-certificate", + "a header with no certificate": "-----BEGIN CERTIFICATE-----", + "an unparseable body": "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----", + "a key rather than a certificate": "-----BEGIN RSA PRIVATE KEY-----\nAAAA\n-----END RSA PRIVATE KEY-----", + "one good and one broken certificate": testCACert + + "\n-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----", + } { + t.Run("rejects a CA bundle that is "+name, func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), caBundle), false) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid caBundle") + assert.Nil(t, cfg) + }) + } + + t.Run("accepts a bundle of several certificates", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + bundle := testCACert + "\n" + testCACert + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), bundle), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Equal(t, bundle, cfg.CaBundle) + }) + + t.Run("still carries the CA bundle when verification is off", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(false), testCACert), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.False(t, cfg.TlsVerify) + assert.Equal(t, testCACert, cfg.CaBundle) + }) +} diff --git a/test/e2e/crds/v1alpha1/gatewayproxy_tls.go b/test/e2e/crds/v1alpha1/gatewayproxy_tls.go new file mode 100644 index 00000000..ec91a92a --- /dev/null +++ b/test/e2e/crds/v1alpha1/gatewayproxy_tls.go @@ -0,0 +1,341 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v1alpha1 + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var _ = Describe("Test GatewayProxy control plane TLS", Label("apisix.apache.org", "v1alpha1", "gatewayproxy"), func() { + var s = scaffold.NewDefaultScaffold() + + // The admin API only listens on plain HTTP, so an openresty in front of it + // stands in for a control plane published over TLS. Its certificate is + // signed by a CA generated per test, which is exactly the case caBundle + // exists for: nothing in the system trust store can verify it. + const adminTLSProxySpec = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: admin-tls +data: + nginx.conf: | + worker_processes 1; + pid /run/nginx.pid; + events { + worker_connections 1024; + } + http { + # the standalone provider PUTs the whole configuration through here + client_max_body_size 32m; + server { + listen 9543 ssl; + ssl_certificate /etc/nginx/ssl/tls.crt; + ssl_certificate_key /etc/nginx/ssl/tls.key; + location / { + proxy_pass %s; + proxy_http_version 1.1; + } + } + server { + listen 9544; + location /healthz { + return 200 'ok'; + } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-tls +spec: + replicas: 1 + selector: + matchLabels: + app: admin-tls + template: + metadata: + labels: + app: admin-tls + spec: + volumes: + - name: config + configMap: + name: admin-tls + - name: ssl + secret: + secretName: admin-tls + containers: + - name: admin-tls + image: "openresty/openresty:1.27.1.2-4-bullseye-fat" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9543 + name: https + protocol: TCP + readinessProbe: + httpGet: + path: /healthz + port: 9544 + initialDelaySeconds: 2 + periodSeconds: 2 + volumeMounts: + - mountPath: /usr/local/openresty/nginx/conf/nginx.conf + name: config + subPath: nginx.conf + - mountPath: /etc/nginx/ssl + name: ssl +--- +apiVersion: v1 +kind: Service +metadata: + name: admin-tls +spec: + selector: + app: admin-tls + ports: + - name: https + port: 9543 + protocol: TCP + targetPort: 9543 +` + + const gatewayProxySpec = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config-tls +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + tlsVerify: true +%s + auth: + type: AdminKey + adminKey: + value: "%s" +` + + const gatewayClassSpec = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: %s +spec: + controllerName: %s +` + + const gatewaySpec = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %s +spec: + gatewayClassName: %s + listeners: + - name: http1 + protocol: HTTP + port: 80 + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config-tls +` + + const httpRouteSpec = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httpbin-tls-cp +spec: + parentRefs: + - name: %s + hostnames: + - "%s" + rules: + - matches: + - path: + type: Exact + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + // indent a PEM block so it survives being inlined into the YAML above + indent := func(pem string) string { + lines := strings.Split(strings.TrimRight(pem, "\n"), "\n") + for i, line := range lines { + lines[i] = " " + line + } + return strings.Join(lines, "\n") + } + + var caCert string + + BeforeEach(func() { + By("generate a CA and a server certificate for the TLS control plane") + // the ADC sidecar dials the proxy by its in-cluster name, so that is + // what the certificate has to be valid for + var serverCert, serverKey string + caCert, serverCert, serverKey = generateSignedCert([]string{ + "admin-tls", + fmt.Sprintf("admin-tls.%s", s.Namespace()), + fmt.Sprintf("admin-tls.%s.svc", s.Namespace()), + }) + + err := s.NewKubeTlsSecret("admin-tls", serverCert, serverKey) + Expect(err).NotTo(HaveOccurred(), "creating the server certificate secret") + + By("deploy a TLS terminator in front of the admin API") + err = s.CreateResourceFromString(fmt.Sprintf(adminTLSProxySpec, s.Deployer.GetAdminEndpoint())) + Expect(err).NotTo(HaveOccurred(), "creating the TLS control plane") + Expect(framework.WaitPodsAvailable(s.GinkgoT, s.KubeOpts(), metav1.ListOptions{ + LabelSelector: "app=admin-tls", + })).NotTo(HaveOccurred(), "waiting for the TLS control plane") + }) + + // routes the gateway through a GatewayProxy that reaches the control plane + // over TLS, with caBundle set to caBundleField + attachRoute := func(hostname, caBundleField string) { + By("create GatewayProxy") + endpoint := fmt.Sprintf("https://admin-tls.%s:9543", s.Namespace()) + err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxySpec, endpoint, caBundleField, s.AdminKey())) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + + By("create GatewayClass") + gatewayClassName := fmt.Sprintf("%s-tls", s.Namespace()) + err = s.CreateResourceFromString(fmt.Sprintf(gatewayClassSpec, gatewayClassName, s.GetControllerName())) + Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") + + By("create Gateway") + gatewayName := fmt.Sprintf("%s-tls", s.Namespace()) + err = s.CreateResourceFromString(fmt.Sprintf(gatewaySpec, gatewayName, gatewayClassName)) + Expect(err).NotTo(HaveOccurred(), "creating Gateway") + + By("create HTTPRoute") + err = s.CreateResourceFromString(fmt.Sprintf(httpRouteSpec, gatewayName, hostname)) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") + } + + It("syncs to a private-CA control plane when caBundle is trusted", func() { + attachRoute("httpbin-tls-cp.org", " caBundle: |\n"+indent(caCert)) + + By("the route is programmed, so the sync got through TLS verification") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin-tls-cp.org", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + }) + + It("fails to sync when caBundle is missing", func() { + attachRoute("httpbin-tls-cp-untrusted.org", "") + + By("the control plane certificate cannot be verified") + // look back as well as forward: the sync can fail before the stream opens + s.WaitControllerManagerLog("unable to verify the first certificate", 60, time.Minute) + + By("so the route is never programmed") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin-tls-cp-untrusted.org", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) + }) + + It("rejects a caBundle that is not a certificate", func() { + By("create GatewayProxy with an unparseable caBundle") + endpoint := fmt.Sprintf("https://admin-tls.%s:9543", s.Namespace()) + output, err := s.CreateResourceFromStringAndGetOutput( + fmt.Sprintf(gatewayProxySpec, endpoint, " caBundle: not-a-certificate", s.AdminKey()), + ) + Expect(err).To(HaveOccurred(), "the API server should reject it") + Expect(output + err.Error()).To(ContainSubstring("caBundle must be a PEM-encoded certificate")) + }) +}) + +// generateSignedCert returns a CA and a server certificate signed by it. The +// scaffold's GenerateMACert gives both the same subject, which OpenSSL reads as +// a self-issued certificate rather than one chaining to the CA, so a strict TLS +// client rejects it before it ever looks at the bundle. +func generateSignedCert(dnsNames []string) (caCertPEM, serverCertPEM, serverKeyPEM string) { + serial := func() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + Expect(err).NotTo(HaveOccurred()) + return n + } + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + caTmpl := &x509.Certificate{ + SerialNumber: serial(), + Subject: pkix.Name{CommonName: "apisix-ingress-e2e-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + + serverKeyECDSA, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + serverTmpl := &x509.Certificate{ + SerialNumber: serial(), + Subject: pkix.Name{CommonName: dnsNames[0]}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: dnsNames, + } + serverDER, err := x509.CreateCertificate(rand.Reader, serverTmpl, caTmpl, &serverKeyECDSA.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + + keyDER, err := x509.MarshalPKCS8PrivateKey(serverKeyECDSA) + Expect(err).NotTo(HaveOccurred()) + + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})), + string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER})), + string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})) +}