From 10e9495950fe64742e3819aab3f93875e5c380ec Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Tue, 1 Sep 2026 18:19:16 +0000 Subject: [PATCH 1/7] Add --cluster-size=size10 postgres footprint option The bundled postgres ships sized for small clusters (--cluster-size=size0, the default): max_connections=100 and modest CPU and memory requests. Larger clusters saturate that and cap ate-api-server well below the load they generate. --cluster-size=size10 merge-patches a tuned postgresql.conf into the postgres configmap, resizes the statefulset to fill a dedicated node, and pins the pgxpool size so the clients open the connections the server is provisioned for. The benchmark orchestrator selects it for the 10k tests. --- hack/install-ate.sh | 72 ++++++++++++++++- .../postgres-config-patch.yaml | 78 +++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 manifests/ate-install/postgres-size10/postgres-config-patch.yaml diff --git a/hack/install-ate.sh b/hack/install-ate.sh index f8f0270c0e..de4eb9215e 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -79,6 +79,7 @@ function usage() { echo " --delete-all Delete core system and all registered demos" echo " --atenet-router=envoy|agentgateway Select the ingress and egress dataplane (default: envoy)" echo " --podcert-workers-per-signer N Concurrent workers per podcertificate-controller signer (default: 1)" + echo " --cluster-size size0|size10 Cluster size profile (default: size0). \"size10\" assumes a dedicated postgres node" echo " --rollout-timeout DURATION Per-workload readiness wait timeout, kubectl-style Go duration (default: 60s)" echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)" echo "" @@ -247,8 +248,26 @@ rollout_timeout() { echo "${timeout}" } +cluster_size() { + local size="${ATE_INSTALL_CLUSTER_SIZE:-size0}" + case "${size}" in + size0|size10) echo "${size}" ;; + *) + echo "Error: --cluster-size must be size0 or size10, got '${size}'" >&2 + exit 1 + ;; + esac +} + default_postgres_connection_string() { - echo "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + local dsn="postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + # pgxpool defaults MaxConns to max(4, runtime.NumCPU()), which under-uses + # the size10 server's raised max_connections. Pin the pool so the client + # side actually opens the sockets the server is provisioned for. + if [[ "$(cluster_size)" == "size10" ]]; then + dsn="${dsn}&pool_max_conns=64&pool_min_conns=4" + fi + echo "${dsn}" } # True if deploying the bundled in-cluster PostgreSQL. Returns false if an @@ -783,6 +802,35 @@ create_api_server_env_vars() { annotate_api_server_env_hash } +apply_postgres_size10_overrides() { + if [[ "$(cluster_size)" != "size10" ]]; then + return 0 + fi + + log_step "apply_postgres_size10_overrides" + + # Merge-patch postgresql.conf; --type merge preserves the other keys + # (pg_hba.conf, reload-tls.sh) that manifests/ate-install/postgres/postgres.yaml + # still owns. + run_kubectl -n ate-system patch configmap postgres-config \ + --type merge \ + --patch-file manifests/ate-install/postgres-size10/postgres-config-patch.yaml + + # Bump the container to fill a dedicated node. Deliberately no CPU + # limit: hostname anti-affinity keeps this pod alone on its + # ate-control-plane node, so a limit only adds CFS throttling on + # checkpoint/autovacuum bursts. Memory request == limit keeps eviction + # ordering equivalent to a Guaranteed pod, which matters because postgres + # cannot release shared_buffers under pressure. Applied via JSON patch so + # the base's cpu limit is actually removed rather than merged. + run_kubectl -n ate-system patch statefulset postgres --type json --patch '[ + {"op":"remove","path":"/spec/template/spec/containers/0/resources/limits/cpu"}, + {"op":"replace","path":"/spec/template/spec/containers/0/resources/requests/cpu","value":"80"}, + {"op":"replace","path":"/spec/template/spec/containers/0/resources/requests/memory","value":"140Gi"}, + {"op":"replace","path":"/spec/template/spec/containers/0/resources/limits/memory","value":"140Gi"} + ]' +} + apply_podcert_workers_override() { if [[ -z "${ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER:-}" ]]; then return 0 @@ -965,6 +1013,10 @@ deploy_ate_system() { ensure_egress_mitm_ca_pool_secret apply_atenet_egress + # Patch postgres before the rollout wait so the wait covers the final + # resized pod, not the size0 base rolled out by render_ate_system_manifests. + apply_postgres_size10_overrides + log_step "Waiting for ATE system components to be ready..." if use_bundled_postgres; then run_kubectl rollout status statefulset/postgres -n ate-system --timeout="$(rollout_timeout)" @@ -1459,6 +1511,14 @@ for ((i = 0; i < ${#prescan_args[@]}; i++)); do fi ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="${prescan_args[$((i + 1))]}" ;; + --cluster-size=*) ATE_INSTALL_CLUSTER_SIZE="${prescan_args[i]#*=}" ;; + --cluster-size) + if (( i + 1 >= ${#prescan_args[@]} )); then + echo "Error: --cluster-size requires size0 or size10" >&2 + exit 1 + fi + ATE_INSTALL_CLUSTER_SIZE="${prescan_args[$((i + 1))]}" + ;; --rollout-timeout=*) ATE_INSTALL_ROLLOUT_TIMEOUT="${prescan_args[i]#*=}" ;; --rollout-timeout) if (( i + 1 >= ${#prescan_args[@]} )); then @@ -1520,6 +1580,7 @@ case "${BENCHMARK_SANDBOX_CLASS}" in ;; esac podcert_workers_per_signer >/dev/null +cluster_size >/dev/null rollout_timeout >/dev/null while [[ "$#" -gt 0 ]]; do @@ -1563,6 +1624,15 @@ while [[ "$#" -gt 0 ]]; do fi ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="$1" ;; + --cluster-size=*) ATE_INSTALL_CLUSTER_SIZE="${1#*=}" ;; + --cluster-size) + shift + if [[ "$#" -eq 0 ]]; then + echo "Error: --cluster-size requires size0 or size10" >&2 + exit 1 + fi + ATE_INSTALL_CLUSTER_SIZE="$1" + ;; --rollout-timeout=*) ATE_INSTALL_ROLLOUT_TIMEOUT="${1#*=}" ;; --rollout-timeout) shift diff --git a/manifests/ate-install/postgres-size10/postgres-config-patch.yaml b/manifests/ate-install/postgres-size10/postgres-config-patch.yaml new file mode 100644 index 0000000000..160d8a1f94 --- /dev/null +++ b/manifests/ate-install/postgres-size10/postgres-config-patch.yaml @@ -0,0 +1,78 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Merge-patched into postgres-config by hack/install-ate.sh when +# --cluster-size=size10. `kubectl patch --type merge` replaces the +# postgresql.conf key without touching pg_hba.conf or reload-tls.sh, both +# defined in manifests/ate-install/postgres/postgres.yaml. +# +# Sized to match the container resource patch in hack/install-ate.sh +# (140 GiB memory, 80 vCPU request, no CPU limit, on a dedicated node). +# Every value below depends on those numbers -- if the container is resized +# the tuning has to move with it. +data: + postgresql.conf: | + # Repeated from manifests/ate-install/postgres/postgres.yaml: kubectl merge patch + # replaces the whole postgresql.conf value, so anything omitted here would + # revert to the compiled-in default. TLS and hba paths have to stay wired + # up or the server refuses connections. + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + + # Memory: container holds 140 GiB. shared_buffers at 25 % is the standard + # starting point; going higher rarely helps on PG 18 because the OS page + # cache double-buffers. effective_cache_size is a planner hint, not an + # allocation. + shared_buffers = 35GB + effective_cache_size = 100GB + work_mem = 32MB + maintenance_work_mem = 2GB + wal_buffers = 64MB + huge_pages = try + + # 2 ate-api-server replicas at pool_max_conns=64 + 3-conn watch pool each + # = 134 steady-state; 300 leaves room to double replicas and for psql / + # monitoring / migrations. + max_connections = 300 + + # 1 GB max_wal_size (the compiled default) forces a checkpoint every few + # seconds under sustained writes and caps throughput well below what the + # disk can do. 16 GB stretches the interval out; completion_target 0.9 + # spreads the flush across it. + max_wal_size = 16GB + min_wal_size = 4GB + checkpoint_completion_target = 0.9 + wal_compression = on + + # Parallelism sized to the container's vCPU (no CPU limit, so it can + # burst to the whole dedicated node). per_gather stays modest because + # the outbox workload is short OLTP, not analytical scans -- unlimited + # parallelism there starves concurrent short queries. + max_worker_processes = 88 + max_parallel_workers = 64 + max_parallel_workers_per_gather = 4 + max_parallel_maintenance_workers = 6 + + # PD-backed SSD assumptions. + random_page_cost = 1.1 + effective_io_concurrency = 200 + + # The outbox pattern churns dead tuples fast; give autovacuum more workers + # and a shorter naptime than the defaults so it stays ahead of writes. + autovacuum_max_workers = 6 + autovacuum_naptime = 10s From 6bad01895a5deda8227d5e779fbf873d08d2fe82 Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Tue, 1 Sep 2026 21:05:45 +0000 Subject: [PATCH 2/7] log command runtimes in install-ate.sh --- hack/install-ate.sh | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index de4eb9215e..f008f4f357 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -19,6 +19,18 @@ set -o errexit -o nounset -o pipefail ROOT="$(git rev-parse --show-toplevel)" cd "${ROOT}" +# _log_elapsed prints wall time since ${1} (an epoch-nanoseconds timestamp +# from `date +%s%N`) to stderr, so wrapping a command in a pipeline does not +# corrupt whatever downstream stage consumes its stdout. Defined here rather +# than with the other helpers below because the gcloud auth block underneath +# uses it. +_log_elapsed() { + local start_ns="$1" label="$2" + local elapsed_ms=$(( ( $(date +%s%N) - start_ns ) / 1000000 )) + printf ' (%s took %d.%03ds)\n' "${label}" \ + "$((elapsed_ms / 1000))" "$((elapsed_ms % 1000))" >&2 +} + # Source the environment variables if configured # TODO: this pattern makes it difficult to switch environments. # Developers will likely want to target both cloud and local depending on what they're working on. @@ -30,7 +42,10 @@ fi if [[ -z "${KUBECTL_CONTEXT:-}" ]]; then # If PROJECT_ID is set, ensure kubeconfig is configured before running any kubectl commands. if [[ -n "${PROJECT_ID:-}" ]]; then + _gcloud_start_ns=$(date +%s%N) gcloud container clusters get-credentials "${CLUSTER_NAME}" --location "${CLUSTER_LOCATION}" --project="${PROJECT_ID}" + _log_elapsed "${_gcloud_start_ns}" "gcloud get-credentials" + unset _gcloud_start_ns fi fi # otherwise just use the current cluster in KUBECONFIG ... @@ -152,9 +167,14 @@ function usage() { } run_kubectl() { + local _start_ns + _start_ns=$(date +%s%N) + local _status=0 kubectl \ ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} \ - "$@" + "$@" || _status=$? + _log_elapsed "${_start_ns}" "kubectl ${1:-}" + return "${_status}" } # run_kubectl_fatal runs kubectl and aborts the install if it fails. Demo @@ -190,12 +210,20 @@ wait_for_pool_rollout_fatal() { } run_kubectl_ate() { + local _start_ns + _start_ns=$(date +%s%N) + local _status=0 go run ./cmd/kubectl-ate \ ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} \ - "$@" + "$@" || _status=$? + _log_elapsed "${_start_ns}" "kubectl-ate ${1:-}" + return "${_status}" } run_ko() { + local _start_ns + _start_ns=$(date +%s%N) + local _status=0 # Build up a set of ldflags to pass to ko. local ldflags=() while IFS= read -r line || [[ -n "${line}" ]]; do @@ -209,13 +237,17 @@ run_ko() { apply|create|delete|run) ./hack/run-tool.sh ko "$@" \ "${ldflags[@]}" \ - ${KUBECTL_CONTEXT:+-- --context="${KUBECTL_CONTEXT}"} + ${KUBECTL_CONTEXT:+-- --context="${KUBECTL_CONTEXT}"} \ + || _status=$? ;; *) ./hack/run-tool.sh ko "$@" \ - "${ldflags[@]}" + "${ldflags[@]}" \ + || _status=$? ;; esac + _log_elapsed "${_start_ns}" "ko ${1:-}" + return "${_status}" } atenet_router() { From a09caa62215da7d0e623d7013b6e4b54e94e044a Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Sun, 13 Sep 2026 06:32:26 +0000 Subject: [PATCH 3/7] install-ate.sh: bound the trust-bundle wait, apply the podcert size10 overlay Bound the install-time wait for the podcertificate-controller's ClusterTrustBundles at 300s and print where to look when it expires, instead of looping forever on a controller that is Ready but not producing bundles. On --cluster-size=size10 clusters install the podcert-size10 kustomize overlay, which raises the controller's client-go rate limits to --kube-api-qps=100 / --kube-api-burst=200 so signing keeps up with the request volume the size10 postgres profile enables. --- hack/install-ate.sh | 46 ++++++++++++++++--- .../podcert-size10/kustomization.yaml | 40 ++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 manifests/ate-install/podcert-size10/kustomization.yaml diff --git a/hack/install-ate.sh b/hack/install-ate.sh index f008f4f357..e4c94e2d24 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -611,11 +611,32 @@ create_podcertificate_controller_cas() { wait_for_podcertificate_trust_bundles() { echo "Waiting for podcertificate ClusterTrustBundles to be ready..." - until run_kubectl get clustertrustbundles podidentity.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do - sleep 1 - done - until run_kubectl get clustertrustbundles servicedns.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do - sleep 1 + local timeout_seconds=300 + local bundles=( + podidentity.podcert.ate.dev:identity:primary-bundle + servicedns.podcert.ate.dev:identity:primary-bundle + ) + # One shared deadline across both bundles so a slow first one doesn't + # eat the full budget of the second, then leave it unbounded. + local deadline=$((SECONDS + timeout_seconds)) + local bundle + for bundle in "${bundles[@]}"; do + until run_kubectl get clustertrustbundles "${bundle}" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + cat >&2 < Date: Fri, 4 Sep 2026 07:18:34 +0000 Subject: [PATCH 4/7] install-ate.sh: tolerate missing CRD kinds during teardown Teardown often runs against a cluster where an earlier upgrade already removed a CRD that manifests/ate-install still references (SandboxConfig, for instance). kubectl delete then fails with "no matches for kind", which --ignore-not-found does not cover, and errexit aborted the rest of the sweep. kubectl_delete_tolerant swallows only that error class. --- hack/install-ate.sh | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index e4c94e2d24..0f455c1cd9 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -209,6 +209,31 @@ wait_for_pool_rollout_fatal() { fi } +# kubectl_delete_tolerant runs `kubectl delete "$@"` and swallows the "no +# matches for kind ..." error class — teardown often runs against a cluster +# where an earlier upgrade already removed a CRD that manifests/ate-install +# still references (e.g. SandboxConfig), and errexit would otherwise abort +# the rest of the sweep. Real errors (any `error:`/`Error` line whose +# message is NOT the missing-kind one) still fail. +kubectl_delete_tolerant() { + local err_tmp status + err_tmp=$(mktemp) + status=0 + run_kubectl delete "$@" 2>"${err_tmp}" || status=$? + cat "${err_tmp}" >&2 + if (( status == 0 )); then + rm -f "${err_tmp}" + return 0 + fi + local other + other=$(grep -E '^(error|Error)' "${err_tmp}" | grep -v 'no matches for kind' || true) + rm -f "${err_tmp}" + if [[ -n "${other}" ]]; then + return "${status}" + fi + return 0 +} + run_kubectl_ate() { local _start_ns _start_ns=$(date +%s%N) @@ -1450,11 +1475,15 @@ delete_substrate_demo() { delete_ate_system() { log_step "delete_ate_system" + # kubectl_delete_tolerant instead of `run_kubectl delete` because + # manifests/ate-install references CRDs (e.g. SandboxConfig) that a prior + # upgrade may have removed from the cluster; --ignore-not-found doesn't + # cover the "no matches for kind" error class. if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then kubectl kustomize manifests/ate-install/kind --load-restrictor LoadRestrictionsNone \ - | run_kubectl delete --ignore-not-found -f - + | kubectl_delete_tolerant --ignore-not-found -f - else - run_kubectl delete --ignore-not-found -f manifests/ate-install + kubectl_delete_tolerant --ignore-not-found -f manifests/ate-install fi run_kubectl delete --ignore-not-found -n ate-system daemonset -l app=atelet From a2aaf804c7ba6fc08e00c583d8928918a68b2349 Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Mon, 31 Aug 2026 18:00:38 +0000 Subject: [PATCH 5/7] up resource counts and cert workers --- manifests/ate-install/ate-api-server.yaml | 4 ++++ manifests/ate-install/ate-controller.yaml | 3 +++ manifests/ate-install/atenet-egress.yaml | 3 +++ manifests/ate-install/atenet-router.yaml | 3 +++ manifests/ate-install/pod-certificate-controller.yaml | 3 +++ manifests/ate-install/postgres/postgres.yaml | 2 +- 6 files changed, 17 insertions(+), 1 deletion(-) diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index cde0c42ab8..28f5c94e74 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -86,6 +86,10 @@ spec: containers: - name: ate-api-server image: ko://github.com/agent-substrate/substrate/cmd/ateapi + resources: + requests: + cpu: "4" + memory: 2Gi args: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - --authentication-config=/etc/ateapi/authentication/authentication.yaml diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 9220cb8d24..7aee95e797 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -102,6 +102,9 @@ spec: containers: - name: ate-controller image: ko://github.com/agent-substrate/substrate/cmd/atecontroller + resources: + requests: + cpu: "4" args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index e36df34f8a..f234573607 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -233,6 +233,9 @@ spec: containers: - name: envoy image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 + resources: + requests: + cpu: "4" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 4f609826e4..d6f5504440 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -143,6 +143,9 @@ spec: containers: - name: atenet-router image: ko://github.com/agent-substrate/substrate/cmd/atenet + resources: + requests: + cpu: "4" args: - "router" # Serve the ingress direction only: the ingress ext_proc handler, plus diff --git a/manifests/ate-install/pod-certificate-controller.yaml b/manifests/ate-install/pod-certificate-controller.yaml index cde4b3809c..5e02731abe 100644 --- a/manifests/ate-install/pod-certificate-controller.yaml +++ b/manifests/ate-install/pod-certificate-controller.yaml @@ -140,6 +140,9 @@ spec: containers: - name: controller image: ko://github.com/agent-substrate/substrate/cmd/podcertcontroller + resources: + requests: + cpu: "4" args: - --in-cluster=true - --sharding-pod-namespace=$(POD_NAMESPACE) diff --git a/manifests/ate-install/postgres/postgres.yaml b/manifests/ate-install/postgres/postgres.yaml index 14a11ea473..d3452f2dc7 100644 --- a/manifests/ate-install/postgres/postgres.yaml +++ b/manifests/ate-install/postgres/postgres.yaml @@ -207,7 +207,7 @@ spec: # down to fit a CI runner. resources: requests: - cpu: "2" + cpu: "4" memory: "1Gi" limits: cpu: "16" From 5a27e7d696af65a298163f3ab44ea1ed9734c24a Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Fri, 4 Sep 2026 19:24:42 +0000 Subject: [PATCH 6/7] Avoid clobbering pod cert controller kustomization --- hack/install-ate.sh | 18 ++++++++++++------ manifests/ate-install/base/kustomization.yaml | 10 +++++++++- manifests/ate-install/kind/kustomization.yaml | 5 ++++- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 0f455c1cd9..943cc88b6d 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -403,8 +403,12 @@ render_ate_system_manifests() { # Build everything resolved with Kustomize for Kind kubectl kustomize manifests/ate-install/kind --load-restrictor LoadRestrictionsNone | run_ko resolve -f - | substitute_version else - # Build everything resolved with base manifests for GKE - run_ko resolve -f manifests/ate-install | substitute_version + # Build everything resolved with the base kustomization for GKE. Not the + # raw directory: that would also re-apply pod-certificate-controller.yaml + # (reverting the size10 flags and the WORKERS_PER_SIGNER override made + # earlier in the install), both atenet-egress variants, and the + # sandboxconfig files, all of which have their own apply steps. + kubectl kustomize manifests/ate-install/base --load-restrictor LoadRestrictionsNone | run_ko resolve -f - | substitute_version fi } @@ -1043,10 +1047,9 @@ deploy_ate_system() { # Ahead of the bundle below, for the same reason as the namespace: every # workload pulls this ConfigMap in via envFrom, and a container whose envFrom - # target is missing will not start. The bundle contains it, but a raw - # directory apply orders by filename, so ate-api-server.yaml and - # ate-controller.yaml would otherwise be created before it and sit in - # CreateContainerConfigError until it caught up. + # target is missing will not start. The bundle contains it too, but applying + # it first keeps the Deployments from depending on the order the renderer + # happens to emit resources in. apply_otel_config ensure_apiserver_prerequisites @@ -1482,6 +1485,9 @@ delete_ate_system() { if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then kubectl kustomize manifests/ate-install/kind --load-restrictor LoadRestrictionsNone \ | kubectl_delete_tolerant --ignore-not-found -f - + # Not part of the kind bundle (see its kustomization), so delete it directly. + # The non-kind branch covers it through the directory delete below. + kubectl_delete_tolerant --ignore-not-found -f manifests/ate-install/pod-certificate-controller.yaml else kubectl_delete_tolerant --ignore-not-found -f manifests/ate-install fi diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 6a2031ef70..a21a9b3276 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -18,10 +18,18 @@ kind: Kustomization # ate-otel-config.yaml carries the OTLP settings every component above consumes # via envFrom. The kind overlay lists its own copy of the same ConfigMap name # instead of building on this directory, so the two never collide. +# +# pod-certificate-controller.yaml is deliberately absent. hack/install-ate.sh +# applies it on its own before this bundle, through the podcert-size10 overlay +# on large clusters, and then sets WORKERS_PER_SIGNER on the Deployment. +# Listing the base file here would re-apply it after those steps and revert +# both the overlay's flags and the env override. The same goes for the +# sandboxconfig and atenet-egress files: install-ate.sh applies the variant it +# selects, so they are not listed here either. resources: - ../ate-api-server.yaml - ../ate-controller.yaml - ../atelet.yaml - ../atenet-router.yaml - - ../pod-certificate-controller.yaml + - ../atenet-router-monitoring.yaml - ../ate-otel-config.yaml diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 4c7a65d95e..38f57fcc93 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -21,12 +21,15 @@ kind: Kustomization # overlay is also built standalone; listing it in both would be a duplicate # resource. hack/install-ate.sh applies the ConfigMap directly for the targeted # single-component redeploys. +# +# pod-certificate-controller.yaml is deliberately absent, as in ../base: +# hack/install-ate.sh applies and deletes it on its own, and re-applying the +# base file from this bundle would revert the WORKERS_PER_SIGNER override. resources: - ../ate-api-server.yaml - ../ate-controller.yaml - ./atelet - ../atenet-router.yaml - - ../pod-certificate-controller.yaml - ate-otel-config.yaml - rustfs.yaml - ./otel-collector.yaml From ef36c72e128f674b92723153fd5aabb446410929 Mon Sep 17 00:00:00 2001 From: Max Smythe Date: Sat, 12 Sep 2026 23:14:26 +0000 Subject: [PATCH 7/7] install-ate.sh: add --cordon-control-plane to pin the control plane to dedicated nodes Under the flag, every control plane workload (ate-api-server, ate-controller, atenet-router, atenet-egress, podcertificate-controller, postgres) gets a nodeSelector and toleration for ate.dev/workloadType=ate-control-plane and a hostname anti-affinity against the other control plane pods, so each runs alone on a node of a dedicated pool. Without the flag the manifests are applied unchanged. The pinning lives in one kustomize component with a name-regex target. The install applies these workloads through several different streams (the base bundle, a lone ate-api-server redeploy, the podcert overlay, the postgres file, and the egress variants), so render_manifests wraps whichever path is in use in a throwaway kustomization that includes the component; kustomize leaves a stream alone when nothing in it matches. --- hack/install-ate.sh | 105 ++++++++++++++---- manifests/ate-install/ate-api-server.yaml | 4 - manifests/ate-install/ate-controller.yaml | 3 - manifests/ate-install/atenet-egress.yaml | 3 - manifests/ate-install/atenet-router.yaml | 3 - .../cordon-control-plane/kustomization.yaml | 67 +++++++++++ .../pod-certificate-controller.yaml | 3 - 7 files changed, 152 insertions(+), 36 deletions(-) create mode 100644 manifests/ate-install/components/cordon-control-plane/kustomization.yaml diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 943cc88b6d..6e6bd37188 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -95,6 +95,9 @@ function usage() { echo " --atenet-router=envoy|agentgateway Select the ingress and egress dataplane (default: envoy)" echo " --podcert-workers-per-signer N Concurrent workers per podcertificate-controller signer (default: 1)" echo " --cluster-size size0|size10 Cluster size profile (default: size0). \"size10\" assumes a dedicated postgres node" + echo " --cordon-control-plane Pin each control plane pod to its own node: assumes a pool labeled and tainted" + echo " ate.dev/workloadType=ate-control-plane:NoSchedule with one node per pod (7 at the" + echo " shipped replica counts) plus a spare, since rollouts surge a new pod first" echo " --rollout-timeout DURATION Per-workload readiness wait timeout, kubectl-style Go duration (default: 60s)" echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)" echo "" @@ -316,6 +319,66 @@ cluster_size() { esac } +cordon_control_plane() { + local cordon="${ATE_INSTALL_CORDON_CONTROL_PLANE:-false}" + case "${cordon}" in + true) return 0 ;; + false) return 1 ;; + *) + echo "Error: --cordon-control-plane must be true or false, got '${cordon}'" >&2 + exit 1 + ;; + esac +} + +# render_manifests emits the manifests at PATH: a plain file is echoed, a +# kustomization directory is built, and "-" reads stdin. Under +# --cordon-control-plane it wraps PATH in a throwaway kustomization that adds +# the cordon-control-plane component, so the same node pinning reaches every +# control plane workload whichever apply path delivers it. The component's +# patch has a name-regex target and kustomize leaves a stream alone when +# nothing in it matches, so wrapping a stream that carries none of those +# workloads is harmless. +render_manifests() { + local path="$1" + if ! cordon_control_plane; then + if [[ "${path}" == "-" ]]; then + cat + elif [[ -d "${path}" ]]; then + kubectl kustomize "${path}" --load-restrictor LoadRestrictionsNone + else + cat "${path}" + fi + return + fi + + local tmp="" + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" RETURN + # kustomize refuses absolute paths as resource or component roots even + # with the load restrictor off, so both are written relative to the + # throwaway directory. + local resource="" + if [[ "${path}" == "-" ]]; then + cat > "${tmp}/stdin.yaml" + resource="stdin.yaml" + else + resource="$(realpath --relative-to="${tmp}" "${path}")" + fi + local component="" + component="$(realpath --relative-to="${tmp}" manifests/ate-install/components/cordon-control-plane)" + cat > "${tmp}/kustomization.yaml" <= ${#prescan_args[@]} )); then @@ -1714,6 +1777,8 @@ while [[ "$#" -gt 0 ]]; do --experimental-use-sdsmint) ;; --experimental-additional-egress-extproc-service) shift ;; --experimental-additional-egress-extproc-service=*) ;; + --cordon-control-plane) ;; + --cordon-control-plane=*) ;; --podcert-workers-per-signer=*) ATE_INSTALL_PODCERT_WORKERS_PER_SIGNER="${1#*=}" ;; --podcert-workers-per-signer) shift diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 28f5c94e74..cde0c42ab8 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -86,10 +86,6 @@ spec: containers: - name: ate-api-server image: ko://github.com/agent-substrate/substrate/cmd/ateapi - resources: - requests: - cpu: "4" - memory: 2Gi args: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - --authentication-config=/etc/ateapi/authentication/authentication.yaml diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 7aee95e797..9220cb8d24 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -102,9 +102,6 @@ spec: containers: - name: ate-controller image: ko://github.com/agent-substrate/substrate/cmd/atecontroller - resources: - requests: - cpu: "4" args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index f234573607..e36df34f8a 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -233,9 +233,6 @@ spec: containers: - name: envoy image: envoyproxy/envoy:v1.39-latest@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 - resources: - requests: - cpu: "4" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index d6f5504440..4f609826e4 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -143,9 +143,6 @@ spec: containers: - name: atenet-router image: ko://github.com/agent-substrate/substrate/cmd/atenet - resources: - requests: - cpu: "4" args: - "router" # Serve the ingress direction only: the ingress ext_proc handler, plus diff --git a/manifests/ate-install/components/cordon-control-plane/kustomization.yaml b/manifests/ate-install/components/cordon-control-plane/kustomization.yaml new file mode 100644 index 0000000000..ffc06dfe1f --- /dev/null +++ b/manifests/ate-install/components/cordon-control-plane/kustomization.yaml @@ -0,0 +1,67 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pins each control plane workload to its own dedicated node. Selected by +# hack/install-ate.sh under --cordon-control-plane, which wraps every +# control plane apply in a kustomization that includes this component. +# +# Assumes a node pool whose nodes carry the label and the taint +# ate.dev/workloadType=ate-control-plane:NoSchedule. The hostname +# anti-affinity matches every pod carrying the label, replicas included, so +# the pool needs one node per pod: 7 at the shipped replica counts (two +# ate-api-server, one of each other workload), and more if replicas are +# raised. Keep at least one spare beyond that: the Deployments surge a new +# pod before removing the old one, and with no free node the new pod stays +# Pending and the rollout never completes. A pool that is too small leaves +# pods Pending rather than failing loudly. +# +# One patch with a name-regex target rather than one per workload: the +# install applies these workloads through several different streams (the +# base bundle, a lone ate-api-server redeploy, the podcert overlay, the +# postgres file, and the egress variants), and kustomize leaves a stream +# alone when nothing in it matches, so the same component fits every path. + +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: + - target: + kind: Deployment|StatefulSet + name: ate-api-server|ate-controller|atenet-router|atenet-egress|podcertificate-controller|postgres + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: overridden-by-target + spec: + template: + metadata: + labels: + ate.dev/workloadType: ate-control-plane + spec: + nodeSelector: + ate.dev/workloadType: ate-control-plane + tolerations: + - key: ate.dev/workloadType + operator: Equal + value: ate-control-plane + effect: NoSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + ate.dev/workloadType: ate-control-plane + namespaceSelector: {} + topologyKey: kubernetes.io/hostname diff --git a/manifests/ate-install/pod-certificate-controller.yaml b/manifests/ate-install/pod-certificate-controller.yaml index 5e02731abe..cde4b3809c 100644 --- a/manifests/ate-install/pod-certificate-controller.yaml +++ b/manifests/ate-install/pod-certificate-controller.yaml @@ -140,9 +140,6 @@ spec: containers: - name: controller image: ko://github.com/agent-substrate/substrate/cmd/podcertcontroller - resources: - requests: - cpu: "4" args: - --in-cluster=true - --sharding-pod-namespace=$(POD_NAMESPACE)