diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index a554c554a2d..15da6c452fb 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -4177,6 +4177,49 @@ SSL Termination :file:`ssl_multicert.yaml` file successfully load. If false (``0``), SSL certificate load failures will not prevent |TS| from starting. +.. ts:cv:: CONFIG proxy.config.ssl.server.multicert.partial_reload INT 0 + :reloadable: + + When set to ``1``, a live ``traffic_ctl config reload`` that encounters one or more + certificate load failures will still commit the partial :cpp:class:`SSLCertLookup` + (containing all certificates that did load cleanly) instead of discarding the entire + new configuration and keeping the old one. + + By default (``0``), the reload is strict and all-or-nothing: any failure causes the + entire new configuration to be rejected and the previous configuration to remain active. + + When the knob is on, any skipped certificate still produces an ``ERROR`` entry in + :file:`diags.log` and increments the + ``proxy.process.ssl.ssl_multicert_load_failures`` metric, so degraded state + is never silent. The metric is also incremented in strict mode (``0``) and can + be used as an alert signal in both configurations. + + This knob takes effect on the next ``traffic_ctl config reload`` without requiring + a process restart. + + .. note:: + + This knob is independent of :ts:cv:`proxy.config.ssl.server.multicert.exit_on_load_fail`. + That option governs startup behavior only. ``partial_reload`` governs live reloads + via :program:`traffic_ctl`. + + .. note:: + + When a partial reload is committed, any hostname whose certificate failed to load + will immediately fall back to the bare TLS bootstrap context (no certificate) rather + than continuing to serve its previously-loaded certificate. Strict mode (``0``) is + safer in this regard, all hostnames continue serving their old certificates until + a fully-successful reload, but at the cost of blocking all certificate updates + whenever any single certificate fails. Enabling ``partial_reload`` trades per-hostname + reliability for reduced blast radius across the certificate set. + + .. note:: + + This knob applies only to TLS server certificate loading (``SSLCertificateConfig``). + The QUIC/HTTP3 certificate loader (``QUICCertConfig``) is not yet covered and continues + to use strict all-or-nothing semantics regardless of this setting. Deployments without + a ``dest_ip: "*"`` wildcard entry (SNI-only configurations) are fully supported. + .. ts:cv:: CONFIG proxy.config.ssl.server.multicert.concurrency INT 1 Controls how many threads are used to load SSL certificates from :file:`ssl_multicert.yaml` diff --git a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst index c3dc1bb7994..6babd8aae7f 100644 --- a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst @@ -115,6 +115,15 @@ SSL/TLS .. ts:stat:: global proxy.process.ssl.ssl_sni_name_set_failure integer :type: counter +.. ts:stat:: global proxy.process.ssl.ssl_multicert_load_failures integer + :type: counter + + Counts certificates that failed to load during any SSL configuration reload. + When :ts:cv:`proxy.config.ssl.server.multicert.partial_reload` is enabled, failed + certificates are skipped and the remaining valid certificates are committed. This + counter increments regardless of that setting and can be used to alert on cert + load failures even in strict (default) mode. + .. ts:stat:: global proxy.process.ssl.total_handshake_time integer :type: counter :units: milliseconds diff --git a/src/iocore/net/P_SSLConfig.h b/src/iocore/net/P_SSLConfig.h index a29755ed64b..87fa8fcdece 100644 --- a/src/iocore/net/P_SSLConfig.h +++ b/src/iocore/net/P_SSLConfig.h @@ -67,6 +67,7 @@ struct SSLConfigParams : public ConfigInfo { char *cipherSuite; char *client_cipherSuite; int configExitOnLoadError; + int configPartialReload; ///< When 1, commit a partial SSLCertLookup on reload even if some certs failed. int configLoadConcurrency; int clientCertLevel; int verify_depth; diff --git a/src/iocore/net/SSLConfig.cc b/src/iocore/net/SSLConfig.cc index 9e8515d1c23..624a2a11c9f 100644 --- a/src/iocore/net/SSLConfig.cc +++ b/src/iocore/net/SSLConfig.cc @@ -130,6 +130,7 @@ SSLConfigParams::reset() ssl_ctx_options = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; ssl_client_ctx_options = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; configExitOnLoadError = 1; + configPartialReload = 0; configLoadConcurrency = 1; } @@ -439,6 +440,7 @@ SSLConfigParams::initialize(ConfigContext ctx) configFilePath = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.server.multicert.filename")); configExitOnLoadError = RecGetRecordInt("proxy.config.ssl.server.multicert.exit_on_load_fail").value_or(0); + configPartialReload = RecGetRecordInt("proxy.config.ssl.server.multicert.partial_reload").value_or(0); configLoadConcurrency = RecGetRecordInt("proxy.config.ssl.server.multicert.concurrency").value_or(1); if (configLoadConcurrency == 0) { configLoadConcurrency = std::clamp(static_cast(std::thread::hardware_concurrency()), 1, 256); @@ -703,8 +705,26 @@ SSLCertificateConfig::reconfigure(ConfigContext ctx) // If the load succeeded, load it. If there is no current configuration, load even a broken // config so that a bad initial load doesn't completely disable TLS. - if (retStatus || configid == 0) { + // If partial_reload is enabled and at least one cert was successfully inserted, commit the + // partial lookup so that a single bad cert does not block all other certs from being updated. + // lookup->count() is used rather than checking ssl_default, because ssl_default is always + // populated with a bare bootstrap context (no X.509) for SNI-only deployments that have no + // dest_ip: "*" entry, checking ssl_default would silently disable partial reload for those. + // count() == 0 means all certs failed: hasAnyCert is false, partialCommit is false, + // the lookup is discarded, and retStatus stays false so traffic_ctl reports failure. + // When count() > 0 and partial_reload is on, retStatus is flipped to true after the + // partial commit so that traffic_ctl reports the reload as successful. + const bool hasAnyCert = lookup->count() > 0; + const bool partialCommit = !retStatus && params->configPartialReload && hasAnyCert; + const bool initialLoad = (configid == 0); + if (retStatus || initialLoad || partialCommit) { configid = configProcessor.set(configid, lookup); + // Only flip retStatus on live reloads (not the initial startup load). + // At startup the lookup is committed unconditionally via the initialLoad branch, and we + // must preserve the false return so that startup() can honour configExitOnLoadError. + if (partialCommit && !initialLoad) { + retStatus = true; + } } else { delete lookup; } diff --git a/src/iocore/net/SSLStats.cc b/src/iocore/net/SSLStats.cc index 2ca32677f0c..583d8238391 100644 --- a/src/iocore/net/SSLStats.cc +++ b/src/iocore/net/SSLStats.cc @@ -260,6 +260,7 @@ SSLInitializeStatistics() ssl_rsb.user_agent_wrong_version = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_wrong_version"); ssl_rsb.tls_handshake_bytes_in_total = Metrics::Counter::createPtr("proxy.process.ssl.total_handshake_bytes_read_in"); ssl_rsb.tls_handshake_bytes_out_total = Metrics::Counter::createPtr("proxy.process.ssl.total_handshake_bytes_write_out"); + ssl_rsb.ssl_multicert_load_failures = Metrics::Counter::createPtr("proxy.process.ssl.ssl_multicert_load_failures"); #if defined(OPENSSL_IS_BORINGSSL) size_t n = SSL_get_all_cipher_names(nullptr, 0); diff --git a/src/iocore/net/SSLStats.h b/src/iocore/net/SSLStats.h index c00371b8253..dbde48a35a1 100644 --- a/src/iocore/net/SSLStats.h +++ b/src/iocore/net/SSLStats.h @@ -128,6 +128,9 @@ struct SSLStatsBlock { Metrics::Gauge::AtomicType *user_agent_session_miss = nullptr; Metrics::Gauge::AtomicType *user_agent_session_timeout = nullptr; Metrics::Gauge::AtomicType *user_agent_sessions = nullptr; + + Metrics::Counter::AtomicType *ssl_multicert_load_failures = + nullptr; ///< Incremented per cert that fails to load during ssl_multicert reload. }; extern SSLStatsBlock ssl_rsb; diff --git a/src/iocore/net/SSLUtils.cc b/src/iocore/net/SSLUtils.cc index 7c7637affa1..d07dd54b300 100644 --- a/src/iocore/net/SSLUtils.cc +++ b/src/iocore/net/SSLUtils.cc @@ -1963,6 +1963,13 @@ SSLMultiCertConfigLoader::_load_items(SSLCertLookup *lookup, config::SSLMultiCer std::lock_guard lock(_loader_mutex); errata.note(ERRATA_ERROR, "Failed to load certificate '{}' at item {}", sslMultiCertSettings->cert ? sslMultiCertSettings->cert : "(unnamed)", item_num); + // Guard required: SSLCertificateConfig::startup() (which calls _load_items()) always + // runs before SSLInitializeStatistics() in the normal startup path (see + // SSLNetProcessor::start()), so this counter is nullptr during the initial cert + // load. Other counters are only incremented from paths that execute after stats init. + if (ssl_rsb.ssl_multicert_load_failures) { + Metrics::Counter::increment(ssl_rsb.ssl_multicert_load_failures); + } } } else { std::lock_guard lock(_loader_mutex); diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index 7890f427ed1..d36964aae29 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -1192,6 +1192,8 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.server.multicert.exit_on_load_fail", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.server.multicert.partial_reload", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , {RECT_CONFIG, "proxy.config.ssl.server.multicert.concurrency", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-256]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.servername.filename", RECD_STRING, ts::filename::SNI, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} diff --git a/tests/gold_tests/tls/ssl_multicert_partial_reload.test.py b/tests/gold_tests/tls/ssl_multicert_partial_reload.test.py new file mode 100644 index 00000000000..c8e4e49e7af --- /dev/null +++ b/tests/gold_tests/tls/ssl_multicert_partial_reload.test.py @@ -0,0 +1,486 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test ssl_multicert partial_reload: with the knob on, a single bad cert should not +block the reload of all other valid certs. +''' + +# --------------------------------------------------------------------------- +# Scenario A - default behavior (partial_reload=0): one bad cert aborts the +# entire reload; old config stays fully in place. +# --------------------------------------------------------------------------- + +sni_valid = 'valid.example.com' + +ts_strict = Test.MakeATSProcess("ts_strict", enable_tls=True, disable_log_checks=True) +server_strict = Test.MakeOriginServer("server_strict") +request_header = {"headers": f"GET / HTTP/1.1\r\nHost: {sni_valid}\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_strict.addResponse("sessionlog.json", request_header, response_header) + +ts_strict.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts_strict.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts_strict.Variables.SSLDir}', + 'proxy.config.ssl.server.multicert.exit_on_load_fail': 0, + # partial_reload intentionally left at default (0) + }) + +ts_strict.addDefaultSSLFiles() +ts_strict.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server_strict.Variables.Port}') + +ts_strict.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + +tr_strict_1 = Test.AddTestRun("Strict: initial request succeeds") +tr_strict_1.Processes.Default.StartBefore(Test.Processes.ts_strict) +tr_strict_1.Processes.Default.StartBefore(server_strict) +tr_strict_1.StillRunningAfter = ts_strict +tr_strict_1.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_strict.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_strict.Variables.ssl_port}", + ts=ts_strict) +tr_strict_1.Processes.Default.ReturnCode = 0 + +# Overwrite config with one bad entry (missing file) + keep the good default +tr_strict_update = Test.AddTestRun("Strict: inject bad cert into config") +strict_yaml_path = ts_strict.Disk.ssl_multicert_yaml.AbsPath +tr_strict_update.Disk.File(strict_yaml_path, id="strict_yaml", typename="ats:config") +tr_strict_update.Disk.strict_yaml.AddLines( + """ +ssl_multicert: + - ssl_cert_name: does_not_exist.pem + ssl_key_name: does_not_exist.key + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) +tr_strict_update.StillRunningAfter = ts_strict +tr_strict_update.Processes.Default.Command = 'echo Updated strict config' +tr_strict_update.Processes.Default.Env = ts_strict.Env +tr_strict_update.Processes.Default.ReturnCode = 0 + +# Reload should fail - strict mode rejects the whole config +tr_strict_reload = Test.AddConfigReload( + ts_strict, expect="fail", expect_tasks=["ssl_multicert.yaml"], description="Strict: reload expected to fail") +tr_strict_reload.StillRunningAfter = server_strict + +# Old cert still served because reload was rolled back +tr_strict_2 = Test.AddTestRun("Strict: old cert still served after failed reload") +tr_strict_2.StillRunningAfter = ts_strict +tr_strict_2.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_strict.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_strict.Variables.ssl_port}", + ts=ts_strict) +tr_strict_2.Processes.Default.ReturnCode = 0 +tr_strict_2.Processes.Default.Streams.stderr = Testers.IncludesExpression( + "CN=example.com", "Old default cert should still be served") + +# The counter must be incremented even in strict mode so operators can alert on failures +# regardless of which reload policy is active. +tr_strict_metric = Test.AddTestRun("Strict: ssl_multicert_load_failures counter incremented") +tr_strict_metric.StillRunningAfter = ts_strict +tr_strict_metric.Processes.Default.Command = ( + f"{ts_strict.Variables.BINDIR}/traffic_ctl metric get" + f" proxy.process.ssl.ssl_multicert_load_failures") +tr_strict_metric.Processes.Default.Env = ts_strict.Env +tr_strict_metric.Processes.Default.ReturnCode = 0 +tr_strict_metric.Processes.Default.Streams.stdout = Testers.IncludesExpression( + "proxy.process.ssl.ssl_multicert_load_failures [1-9]", + "Failure counter must be incremented even when strict mode rolls back the config") + +# --------------------------------------------------------------------------- +# Scenario B - partial_reload=1: a bad cert is skipped, valid certs are +# applied. The domain whose cert failed falls back to the default context. +# --------------------------------------------------------------------------- + +ts_partial = Test.MakeATSProcess("ts_partial", enable_tls=True, disable_log_checks=True) +server_partial = Test.MakeOriginServer("server_partial") +server_partial.addResponse("sessionlog.json", request_header, response_header) + +ts_partial.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts_partial.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts_partial.Variables.SSLDir}', + 'proxy.config.ssl.server.multicert.exit_on_load_fail': 0, + 'proxy.config.ssl.server.multicert.partial_reload': 1, + }) + +ts_partial.addDefaultSSLFiles() +ts_partial.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server_partial.Variables.Port}') + +ts_partial.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + +tr_partial_1 = Test.AddTestRun("Partial: initial request succeeds") +tr_partial_1.Processes.Default.StartBefore(Test.Processes.ts_partial) +tr_partial_1.Processes.Default.StartBefore(server_partial) +tr_partial_1.StillRunningAfter = ts_partial +tr_partial_1.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_partial.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_partial.Variables.ssl_port}", + ts=ts_partial) +tr_partial_1.Processes.Default.ReturnCode = 0 + +# Inject a mix: one bad entry, one NEWLY-GENERATED good default (different CN) +# The new default cert has CN=reloaded.example.com so that after the partial reload +# we can verify the new config was committed rather than the old one retained. +tr_partial_update = Test.AddTestRun("Partial: inject bad cert alongside new valid default") +partial_yaml_path = ts_partial.Disk.ssl_multicert_yaml.AbsPath +ssl_dir = ts_partial.Variables.SSLDir + +# Generate a new self-signed cert with a distinct CN in the same SSL dir +new_cert = f"{ssl_dir}/newdefault.pem" +new_key = f"{ssl_dir}/newdefault.key" +gen_cmd = ( + f"openssl req -x509 -newkey rsa:2048 " + f"-keyout {new_key} -out {new_cert} " + f"-days 365 -nodes -subj '/CN=reloaded.example.com' " + f"2>/dev/null") + +tr_partial_update.Disk.File(partial_yaml_path, id="partial_yaml", typename="ats:config") +tr_partial_update.Disk.partial_yaml.AddLines( + f""" +ssl_multicert: + - ssl_cert_name: does_not_exist.pem + ssl_key_name: does_not_exist.key + - dest_ip: "*" + ssl_cert_name: newdefault.pem + ssl_key_name: newdefault.key +""".split("\n")) +tr_partial_update.StillRunningAfter = ts_partial +tr_partial_update.Processes.Default.Command = f"{gen_cmd} && echo Updated partial config" +tr_partial_update.Processes.Default.Env = ts_partial.Env +tr_partial_update.Processes.Default.ReturnCode = 0 + +# Reload should succeed despite the bad entry (partial_reload=1) +tr_partial_reload = Test.AddConfigReload( + ts_partial, + expect="success", + expect_tasks=["ssl_multicert.yaml"], + description="Partial: reload expected to succeed despite bad cert") +tr_partial_reload.StillRunningAfter = server_partial + +# Default cert is still reachable and the CN proves the NEW config was committed, +# not the old server.pem (CN=example.com) that was kept. +tr_partial_2 = Test.AddTestRun("Partial: new cert committed and served after partial reload") +tr_partial_2.StillRunningAfter = ts_partial +tr_partial_2.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_partial.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_partial.Variables.ssl_port}", + ts=ts_partial) +tr_partial_2.Processes.Default.ReturnCode = 0 +tr_partial_2.Processes.Default.Streams.stderr = Testers.IncludesExpression( + "CN=reloaded.example.com", "New default cert (not old server.pem) must be served") + +# Match the specific loader error message so the assertion fails if the wrong +# (or no) cert-load error was logged, not merely any unrelated ERROR entry. +# The actual diags.log message comes from SSLError() in SSLUtils.cc: +# "failed to load certificate secret for /does_not_exist.pem ..." +ts_partial.Disk.diags_log.Content = Testers.IncludesExpression( + "failed to load certificate secret for.*does_not_exist\\.pem", + "bad cert should produce a specific cert-load error in diags.log") + +# Verify the ssl_multicert_load_failures metric was incremented +tr_metric = Test.AddTestRun("Partial: ssl_multicert_load_failures metric incremented") +tr_metric.StillRunningAfter = ts_partial +tr_metric.Processes.Default.Command = ( + f"{ts_partial.Variables.BINDIR}/traffic_ctl metric get" + f" proxy.process.ssl.ssl_multicert_load_failures") +tr_metric.Processes.Default.Env = ts_partial.Env +tr_metric.Processes.Default.ReturnCode = 0 +tr_metric.Processes.Default.Streams.stdout = Testers.IncludesExpression( + "proxy.process.ssl.ssl_multicert_load_failures [1-9]", + "Failure counter must be at least 1 after a partial reload with a bad cert") + +# --------------------------------------------------------------------------- +# Scenario C - SNI-only deployment (no dest_ip: "*"): partial_reload=1 must +# still commit the good certs even when there is no wildcard default entry. +# This covers CDN-style configurations that rely purely on SNI matching. +# --------------------------------------------------------------------------- + +ts_sni_only = Test.MakeATSProcess("ts_sni_only", enable_tls=True, disable_log_checks=True) +server_sni_only = Test.MakeOriginServer("server_sni_only") +server_sni_only.addResponse("sessionlog.json", request_header, response_header) + +ts_sni_only.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts_sni_only.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts_sni_only.Variables.SSLDir}', + 'proxy.config.ssl.server.multicert.exit_on_load_fail': 0, + 'proxy.config.ssl.server.multicert.partial_reload': 1, + }) + +ts_sni_only.addDefaultSSLFiles() +ts_sni_only.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server_sni_only.Variables.Port}') + +# SNI-only config: server.pem is listed without dest_ip: "*" so it is matched +# by CN/SAN only. ATS will create a bare bootstrap context internally. +ts_sni_only.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + +tr_sni_only_1 = Test.AddTestRun("SNI-only: initial request succeeds") +tr_sni_only_1.Processes.Default.StartBefore(Test.Processes.ts_sni_only) +tr_sni_only_1.Processes.Default.StartBefore(server_sni_only) +tr_sni_only_1.StillRunningAfter = ts_sni_only +tr_sni_only_1.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_sni_only.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_sni_only.Variables.ssl_port}", + ts=ts_sni_only) +tr_sni_only_1.Processes.Default.ReturnCode = 0 + +# Update: add a non-existent cert alongside the good SNI cert (still no dest_ip: "*") +tr_sni_only_update = Test.AddTestRun("SNI-only: inject bad cert into SNI-only config") +sni_yaml_path = ts_sni_only.Disk.ssl_multicert_yaml.AbsPath +tr_sni_only_update.Disk.File(sni_yaml_path, id="sni_yaml", typename="ats:config") +tr_sni_only_update.Disk.sni_yaml.AddLines( + """ +ssl_multicert: + - ssl_cert_name: does_not_exist.pem + ssl_key_name: does_not_exist.key + - ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) +tr_sni_only_update.StillRunningAfter = ts_sni_only +tr_sni_only_update.Processes.Default.Command = 'echo Updated SNI-only config' +tr_sni_only_update.Processes.Default.Env = ts_sni_only.Env +tr_sni_only_update.Processes.Default.ReturnCode = 0 + +# Reload must succeed: lookup->count() > 0 because server.pem was inserted even +# though ssl_default holds only the bare bootstrap context (no X.509 cert). +tr_sni_only_reload = Test.AddConfigReload( + ts_sni_only, + expect="success", + expect_tasks=["ssl_multicert.yaml"], + description="SNI-only: partial reload succeeds even without dest_ip: '*' entry") +tr_sni_only_reload.StillRunningAfter = server_sni_only + +# The good SNI cert must still be reachable after the partial reload +tr_sni_only_2 = Test.AddTestRun("SNI-only: good cert still served after partial reload") +tr_sni_only_2.StillRunningAfter = ts_sni_only +tr_sni_only_2.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_sni_only.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_sni_only.Variables.ssl_port}", + ts=ts_sni_only) +tr_sni_only_2.Processes.Default.ReturnCode = 0 +tr_sni_only_2.Processes.Default.Streams.stderr = Testers.IncludesExpression( + "CN=example.com", "server.pem (CN=example.com) must still be served after partial reload") + +tr_sni_only_metric = Test.AddTestRun("SNI-only: ssl_multicert_load_failures metric incremented") +tr_sni_only_metric.StillRunningAfter = ts_sni_only +tr_sni_only_metric.Processes.Default.Command = ( + f"{ts_sni_only.Variables.BINDIR}/traffic_ctl metric get" + f" proxy.process.ssl.ssl_multicert_load_failures") +tr_sni_only_metric.Processes.Default.Env = ts_sni_only.Env +tr_sni_only_metric.Processes.Default.ReturnCode = 0 +tr_sni_only_metric.Processes.Default.Streams.stdout = Testers.IncludesExpression( + "proxy.process.ssl.ssl_multicert_load_failures [1-9]", + "Failure counter must be incremented for the skipped cert in SNI-only mode") + +# --------------------------------------------------------------------------- +# Scenario D - dest_ip: "*" fails but SNI-specific cert succeeds: partial +# commit must still occur because lookup->count() > 0 (the SNI cert was +# inserted). +# --------------------------------------------------------------------------- + +ts_d = Test.MakeATSProcess("ts_d", enable_tls=True, disable_log_checks=True) +server_d = Test.MakeOriginServer("server_d") +server_d.addResponse("sessionlog.json", request_header, response_header) + +ts_d.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts_d.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts_d.Variables.SSLDir}', + 'proxy.config.ssl.server.multicert.exit_on_load_fail': 0, + 'proxy.config.ssl.server.multicert.partial_reload': 1, + }) + +ts_d.addDefaultSSLFiles() +ts_d.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server_d.Variables.Port}') +ssl_dir_d = ts_d.Variables.SSLDir + +# Initial config: both a wildcard default and a SNI-specific cert +ts_d.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key + - ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + +tr_d_1 = Test.AddTestRun("Default-fails: initial request succeeds") +tr_d_1.Processes.Default.StartBefore(Test.Processes.ts_d) +tr_d_1.Processes.Default.StartBefore(server_d) +tr_d_1.StillRunningAfter = ts_d +tr_d_1.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_d.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_d.Variables.ssl_port}", + ts=ts_d) +tr_d_1.Processes.Default.ReturnCode = 0 + +# Update: break the dest_ip: "*" entry, keep the SNI cert valid +# A distinct SNI cert is generated so we can confirm it was committed. +tr_d_update = Test.AddTestRun("Default-fails: break wildcard default, keep SNI cert valid") +d_yaml_path = ts_d.Disk.ssl_multicert_yaml.AbsPath +new_sni_cert_d = f"{ssl_dir_d}/sni_cert_d.pem" +new_sni_key_d = f"{ssl_dir_d}/sni_cert_d.key" +gen_sni_cmd_d = ( + f"openssl req -x509 -newkey rsa:2048 " + f"-keyout {new_sni_key_d} -out {new_sni_cert_d} " + f"-days 365 -nodes -subj '/CN=sni-d.example.com' " + f"2>/dev/null") +tr_d_update.Disk.File(d_yaml_path, id="d_yaml", typename="ats:config") +tr_d_update.Disk.d_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: does_not_exist.pem + ssl_key_name: does_not_exist.key + - ssl_cert_name: sni_cert_d.pem + ssl_key_name: sni_cert_d.key +""".split("\n")) +tr_d_update.StillRunningAfter = ts_d +tr_d_update.Processes.Default.Command = f"{gen_sni_cmd_d} && echo Updated default-fails config" +tr_d_update.Processes.Default.Env = ts_d.Env +tr_d_update.Processes.Default.ReturnCode = 0 + +# Reload must succeed: sni_cert_d.pem was inserted so lookup->count() > 0. +tr_d_reload = Test.AddConfigReload( + ts_d, + expect="success", + expect_tasks=["ssl_multicert.yaml"], + description="Default-fails: partial reload succeeds because SNI cert is healthy") +tr_d_reload.StillRunningAfter = server_d + +tr_d_metric = Test.AddTestRun("Default-fails: ssl_multicert_load_failures metric incremented") +tr_d_metric.StillRunningAfter = ts_d +tr_d_metric.Processes.Default.Command = ( + f"{ts_d.Variables.BINDIR}/traffic_ctl metric get" + f" proxy.process.ssl.ssl_multicert_load_failures") +tr_d_metric.Processes.Default.Env = ts_d.Env +tr_d_metric.Processes.Default.ReturnCode = 0 +tr_d_metric.Processes.Default.Streams.stdout = Testers.IncludesExpression( + "proxy.process.ssl.ssl_multicert_load_failures [1-9]", + "Failure counter must be incremented when the wildcard default cert fails") + +# --------------------------------------------------------------------------- +# Scenario E - all certs fail with partial_reload=1: lookup->count()==0 so +# hasAnyCert is false, partialCommit is false, reload must be reported as +# failed and the old configuration must remain active. +# This locks in the behavior that the comment in SSLConfig.cc documents: +# count()==0 keeps retStatus false, reload is not silently reported as +# "success" when there is literally nothing new to serve. +# --------------------------------------------------------------------------- + +ts_e = Test.MakeATSProcess("ts_e", enable_tls=True, disable_log_checks=True) +server_e = Test.MakeOriginServer("server_e") +server_e.addResponse("sessionlog.json", request_header, response_header) + +ts_e.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts_e.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts_e.Variables.SSLDir}', + 'proxy.config.ssl.server.multicert.exit_on_load_fail': 0, + 'proxy.config.ssl.server.multicert.partial_reload': 1, + }) + +ts_e.addDefaultSSLFiles() +ts_e.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server_e.Variables.Port}') + +# Initial config: one valid default cert so ATS starts successfully. +ts_e.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + +tr_e_1 = Test.AddTestRun("All-fail: initial request succeeds") +tr_e_1.Processes.Default.StartBefore(Test.Processes.ts_e) +tr_e_1.Processes.Default.StartBefore(server_e) +tr_e_1.StillRunningAfter = ts_e +tr_e_1.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_e.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_e.Variables.ssl_port}", + ts=ts_e) +tr_e_1.Processes.Default.ReturnCode = 0 + +# Replace config with ONLY non-existent certs, every entry will fail, count()==0. +tr_e_update = Test.AddTestRun("All-fail: replace config with entirely invalid certs") +e_yaml_path = ts_e.Disk.ssl_multicert_yaml.AbsPath +tr_e_update.Disk.File(e_yaml_path, id="e_yaml", typename="ats:config") +tr_e_update.Disk.e_yaml.AddLines( + """ +ssl_multicert: + - ssl_cert_name: does_not_exist_1.pem + ssl_key_name: does_not_exist_1.key + - ssl_cert_name: does_not_exist_2.pem + ssl_key_name: does_not_exist_2.key +""".split("\n")) +tr_e_update.StillRunningAfter = ts_e +tr_e_update.Processes.Default.Command = 'echo Updated all-fail config' +tr_e_update.Processes.Default.Env = ts_e.Env +tr_e_update.Processes.Default.ReturnCode = 0 + +# Reload must fail: count()==0 means hasAnyCert=false, partialCommit=false, +# the lookup is discarded and retStatus stays false. +tr_e_reload = Test.AddConfigReload( + ts_e, + expect="fail", + expect_tasks=["ssl_multicert.yaml"], + description="All-fail: reload must fail even with partial_reload=1 when count()==0") +tr_e_reload.StillRunningAfter = server_e + +# Old cert must still be served because the reload was rolled back. +tr_e_2 = Test.AddTestRun("All-fail: old cert still served after all-fail reload") +tr_e_2.StillRunningAfter = ts_e +tr_e_2.MakeCurlCommand( + f"-q -s -v -k --resolve '{sni_valid}:{ts_e.Variables.ssl_port}:127.0.0.1' " + f"https://{sni_valid}:{ts_e.Variables.ssl_port}", + ts=ts_e) +tr_e_2.Processes.Default.ReturnCode = 0 +tr_e_2.Processes.Default.Streams.stderr = Testers.IncludesExpression( + "CN=example.com", "Old default cert must still be served when all new certs failed") + +# Counter must be incremented for each cert that failed, even though nothing was committed. +tr_e_metric = Test.AddTestRun("All-fail: ssl_multicert_load_failures counter incremented") +tr_e_metric.StillRunningAfter = ts_e +tr_e_metric.Processes.Default.Command = ( + f"{ts_e.Variables.BINDIR}/traffic_ctl metric get" + f" proxy.process.ssl.ssl_multicert_load_failures") +tr_e_metric.Processes.Default.Env = ts_e.Env +tr_e_metric.Processes.Default.ReturnCode = 0 +tr_e_metric.Processes.Default.Streams.stdout = Testers.IncludesExpression( + "proxy.process.ssl.ssl_multicert_load_failures [1-9]", + "Failure counter must be incremented for each cert that could not be loaded")