From 36c45067dea9dfe32d54d21f3b054028041b0a88 Mon Sep 17 00:00:00 2001
From: vitostrokov <57577597+vitostrokov@users.noreply.github.com>
Date: Sun, 18 Oct 2026 08:29:40 +0100
Subject: [PATCH] Resend gateway config before re-registering services

The running gateway replica pipeline re-registers services after a
gateway loses its state, e.g. when the gateway is reinstalled, but it
never resends the global config. Certificates for the re-registered
services are then requested from certbot's default ACME server instead
of the configured one (staging, or an EAB CA). Only a server restart
resent the config, through init_gateways.

Submit the config before any service registration. The call is
idempotent and happens only on ticks that register services. If it
fails, skip the sync until the next tick rather than issue a
certificate from the wrong server.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../pipeline_tasks/gateway_replicas.py        | 12 +++++
 .../pipeline_tasks/test_gateway_replicas.py   | 47 +++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py
index 7e51f90eff..2530e10ec3 100644
--- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py
+++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py
@@ -708,6 +708,18 @@ async def _process_running_item(item: GatewayReplicaPipelineItem):
             session, plan
         )
 
+    if plan.register_services:
+        # Registering a service may request a TLS certificate. A reinstalled gateway has
+        # lost its global config along with its services, and certbot would fall back to
+        # its default ACME server, so resend the config first.
+        try:
+            async with connection.client() as client:
+                await client.submit_gateway_config()
+        except Exception as e:
+            logger.warning("%s: failed to submit gateway config: %r", fmt(replica_model), e)
+            await _commit_update(item, replica_model, update_map={})
+            return
+
     sync_result = await _perform_state_sync(
         connection, gateway_model, replica_model, run_models_by_id, job_models_by_id, plan
     )
diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py
index aaddd0b7eb..1c1ca8bb4f 100644
--- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py
+++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py
@@ -878,6 +878,9 @@ async def test_registers_new_service_and_replica(
             ssh_head_proxy_private_key=None,
         )
         assert client_mock.register_replica.call_args.kwargs["job_submission"].id == job.id
+        # The config (ACME settings) is resent before registration requests a certificate
+        called = [c[0] for c in client_mock.method_calls]
+        assert called.index("submit_gateway_config") < called.index("register_service")
         client_mock.unregister_service.assert_not_called()
         client_mock.unregister_replica.assert_not_called()
 
@@ -909,6 +912,49 @@ async def test_registers_new_service_and_replica(
             "Service replica registered",
         }
 
+    async def test_skips_registration_if_gateway_config_not_submitted(
+        self,
+        test_db,
+        session: AsyncSession,
+        worker: GatewayReplicaWorker,
+        mock_gateway_connection: AsyncMock,
+    ):
+        project = await create_project(session=session)
+        user = await create_user(session=session)
+        repo = await create_repo(session=session, project_id=project.id)
+        backend = await create_backend(session=session, project_id=project.id)
+        gateway = await create_gateway(
+            session=session,
+            project_id=project.id,
+            backend_id=backend.id,
+            status=GatewayStatus.RUNNING,
+        )
+        replica = await create_gateway_replica(
+            session=session,
+            gateway_id=gateway.id,
+            backend=backend,
+            status=GatewayReplicaStatus.RUNNING,
+        )
+        await self._create_service_run_and_job(
+            session, project, repo, user, gateway, run_name="test-service"
+        )
+        _lock_replica(replica)
+        await session.commit()
+
+        client_mock = _get_client_mock(mock_gateway_connection)
+        client_mock.list_services.return_value = []
+        client_mock.submit_gateway_config.side_effect = RuntimeError("gateway unavailable")
+
+        await worker.process(_replica_to_pipeline_item(replica))
+
+        # Registering now could issue a certificate from the wrong ACME server,
+        # so the sync waits for the next tick
+        client_mock.register_service.assert_not_called()
+        client_mock.register_replica.assert_not_called()
+        await session.refresh(replica)
+        assert replica.status == GatewayReplicaStatus.RUNNING
+        assert replica.lock_token is None
+
     async def test_unregisters_dangling_service_and_stale_replica(
         self,
         test_db,
@@ -1304,6 +1350,7 @@ async def test_no_gateway_calls_when_state_already_in_sync(
         client_mock.unregister_service.assert_not_called()
         client_mock.unregister_replica.assert_not_called()
         client_mock.set_service_id.assert_not_called()
+        client_mock.submit_gateway_config.assert_not_called()
 
         # Registration records are still (re)created to reflect the confirmed state,
         # even though no gateway calls were needed this tick.