TLS: add proxy.config.ssl.server.multicert.partial_reload

doc/admin-guide/files/records.yaml.en.rst

@@ -4177,6 +4177,49 @@ SSL Termination
    :file:`ssl_multicert.yaml` file successfully load.  If false (``0``), SSL certificate
    load failures will not prevent |TS| from starting.
 
+.. ts:cv:: CONFIG proxy.config.ssl.server.multicert.partial_reload INT 0
+   :reloadable:
+
+   When set to ``1``, a live ``traffic_ctl config reload`` that encounters one or more
+   certificate load failures will still commit the partial :cpp:class:`SSLCertLookup`
+   (containing all certificates that did load cleanly) instead of discarding the entire
+   new configuration and keeping the old one.
+
+   By default (``0``), the reload is strict and all-or-nothing: any failure causes the
+   entire new configuration to be rejected and the previous configuration to remain active.
+
+   When the knob is on, any skipped certificate still produces an ``ERROR`` entry in
+   :file:`diags.log` and increments the
+   ``proxy.process.ssl.ssl_multicert_load_failures`` metric, so degraded state
+   is never silent.  The metric is also incremented in strict mode (``0``) and can
+   be used as an alert signal in both configurations.
+
+   This knob takes effect on the next ``traffic_ctl config reload`` without requiring
+   a process restart.
+
+   .. note::
+
+      This knob is independent of :ts:cv:`proxy.config.ssl.server.multicert.exit_on_load_fail`.
+      That option governs startup behavior only.  ``partial_reload`` governs live reloads
+      via :program:`traffic_ctl`.
+
+   .. note::
+
+      When a partial reload is committed, any hostname whose certificate failed to load
+      will immediately fall back to the bare TLS bootstrap context (no certificate) rather
+      than continuing to serve its previously-loaded certificate.  Strict mode (``0``) is
+      safer in this regard, all hostnames continue serving their old certificates until
+      a fully-successful reload, but at the cost of blocking all certificate updates
+      whenever any single certificate fails.  Enabling ``partial_reload`` trades per-hostname
+      reliability for reduced blast radius across the certificate set.
+
+   .. note::
+
+      This knob applies only to TLS server certificate loading (``SSLCertificateConfig``).
+      The QUIC/HTTP3 certificate loader (``QUICCertConfig``) is not yet covered and continues
+      to use strict all-or-nothing semantics regardless of this setting.  Deployments without
+      a ``dest_ip: "*"`` wildcard entry (SNI-only configurations) are fully supported.
+
 .. ts:cv:: CONFIG proxy.config.ssl.server.multicert.concurrency INT 1
 
    Controls how many threads are used to load SSL certificates from :file:`ssl_multicert.yaml`

doc/admin-guide/monitoring/statistics/core/ssl.en.rst

@@ -115,6 +115,15 @@ SSL/TLS
 .. ts:stat:: global proxy.process.ssl.ssl_sni_name_set_failure integer
    :type: counter
 
+.. ts:stat:: global proxy.process.ssl.ssl_multicert_load_failures integer
+   :type: counter
+
+   Counts certificates that failed to load during any SSL configuration reload.
+   When :ts:cv:`proxy.config.ssl.server.multicert.partial_reload` is enabled, failed
+   certificates are skipped and the remaining valid certificates are committed.  This
+   counter increments regardless of that setting and can be used to alert on cert
+   load failures even in strict (default) mode.
+
 .. ts:stat:: global proxy.process.ssl.total_handshake_time integer
    :type: counter
    :units: milliseconds

src/iocore/net/P_SSLConfig.h

@@ -67,6 +67,7 @@ struct SSLConfigParams : public ConfigInfo {
   char *cipherSuite;
   char *client_cipherSuite;
   int   configExitOnLoadError;
+  int   configPartialReload; ///< When 1, commit a partial SSLCertLookup on reload even if some certs failed.
   int   configLoadConcurrency;
   int   clientCertLevel;
   int   verify_depth;

src/iocore/net/SSLConfig.cc

@@ -130,6 +130,7 @@ SSLConfigParams::reset()
   ssl_ctx_options                                      = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
   ssl_client_ctx_options                               = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
   configExitOnLoadError                                = 1;
+  configPartialReload                                  = 0;
   configLoadConcurrency                                = 1;
 }
 
@@ -439,6 +440,7 @@ SSLConfigParams::initialize(ConfigContext ctx)
 
   configFilePath        = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.server.multicert.filename"));
   configExitOnLoadError = RecGetRecordInt("proxy.config.ssl.server.multicert.exit_on_load_fail").value_or(0);
+  configPartialReload   = RecGetRecordInt("proxy.config.ssl.server.multicert.partial_reload").value_or(0);
   configLoadConcurrency = RecGetRecordInt("proxy.config.ssl.server.multicert.concurrency").value_or(1);
   if (configLoadConcurrency == 0) {
     configLoadConcurrency = std::clamp(static_cast<int>(std::thread::hardware_concurrency()), 1, 256);
@@ -703,8 +705,26 @@ SSLCertificateConfig::reconfigure(ConfigContext ctx)
 
   // If the load succeeded, load it. If there is no current configuration, load even a broken
   // config so that a bad initial load doesn't completely disable TLS.
-  if (retStatus || configid == 0) {
+  // If partial_reload is enabled and at least one cert was successfully inserted, commit the
+  // partial lookup so that a single bad cert does not block all other certs from being updated.
+  // lookup->count() is used rather than checking ssl_default, because ssl_default is always
+  // populated with a bare bootstrap context (no X.509) for SNI-only deployments that have no
+  // dest_ip: "*" entry, checking ssl_default would silently disable partial reload for those.
+  // count() == 0 means all certs failed: hasAnyCert is false, partialCommit is false,
+  // the lookup is discarded, and retStatus stays false so traffic_ctl reports failure.
+  // When count() > 0 and partial_reload is on, retStatus is flipped to true after the
+  // partial commit so that traffic_ctl reports the reload as successful.
+  const bool hasAnyCert    = lookup->count() > 0;
+  const bool partialCommit = !retStatus && params->configPartialReload && hasAnyCert;
+  const bool initialLoad   = (configid == 0);
+  if (retStatus || initialLoad || partialCommit) {
     configid = configProcessor.set(configid, lookup);
+    // Only flip retStatus on live reloads (not the initial startup load).
+    // At startup the lookup is committed unconditionally via the initialLoad branch, and we
+    // must preserve the false return so that startup() can honour configExitOnLoadError.
+    if (partialCommit && !initialLoad) {
+      retStatus = true;
+    }
   } else {
     delete lookup;
   }

src/iocore/net/SSLStats.cc

@@ -260,6 +260,7 @@ SSLInitializeStatistics()
   ssl_rsb.user_agent_wrong_version          = Metrics::Counter::createPtr("proxy.process.ssl.user_agent_wrong_version");
   ssl_rsb.tls_handshake_bytes_in_total      = Metrics::Counter::createPtr("proxy.process.ssl.total_handshake_bytes_read_in");
   ssl_rsb.tls_handshake_bytes_out_total     = Metrics::Counter::createPtr("proxy.process.ssl.total_handshake_bytes_write_out");
+  ssl_rsb.ssl_multicert_load_failures       = Metrics::Counter::createPtr("proxy.process.ssl.ssl_multicert_load_failures");
 
 #if defined(OPENSSL_IS_BORINGSSL)
   size_t                    n = SSL_get_all_cipher_names(nullptr, 0);

src/iocore/net/SSLStats.h

@@ -128,6 +128,9 @@ struct SSLStatsBlock {
   Metrics::Gauge::AtomicType *user_agent_session_miss    = nullptr;
   Metrics::Gauge::AtomicType *user_agent_session_timeout = nullptr;
   Metrics::Gauge::AtomicType *user_agent_sessions        = nullptr;
+
+  Metrics::Counter::AtomicType *ssl_multicert_load_failures =
+    nullptr; ///< Incremented per cert that fails to load during ssl_multicert reload.
 };
 
 extern SSLStatsBlock                                                   ssl_rsb;

src/iocore/net/SSLUtils.cc

@@ -1963,6 +1963,13 @@ SSLMultiCertConfigLoader::_load_items(SSLCertLookup *lookup, config::SSLMultiCer
         std::lock_guard<std::mutex> lock(_loader_mutex);
         errata.note(ERRATA_ERROR, "Failed to load certificate '{}' at item {}",
                     sslMultiCertSettings->cert ? sslMultiCertSettings->cert : "(unnamed)", item_num);
+        // Guard required: SSLCertificateConfig::startup() (which calls _load_items()) always
+        // runs before SSLInitializeStatistics() in the normal startup path (see
+        // SSLNetProcessor::start()), so this counter is nullptr during the initial cert
+        // load. Other counters are only incremented from paths that execute after stats init.
+        if (ssl_rsb.ssl_multicert_load_failures) {
+          Metrics::Counter::increment(ssl_rsb.ssl_multicert_load_failures);
+        }
       }
     } else {
       std::lock_guard<std::mutex> lock(_loader_mutex);

src/records/RecordsConfig.cc

@@ -1192,6 +1192,8 @@ static constexpr RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.ssl.server.multicert.exit_on_load_fail", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.ssl.server.multicert.partial_reload", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.ssl.server.multicert.concurrency", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-256]", RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.ssl.servername.filename", RECD_STRING, ts::filename::SNI, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}