diff --git a/mkdocs/docs/concepts/metrics.md b/mkdocs/docs/concepts/metrics.md index 3bebef4c09..889c7bc9b7 100644 --- a/mkdocs/docs/concepts/metrics.md +++ b/mkdocs/docs/concepts/metrics.md @@ -190,18 +190,6 @@ telemetry, and more. ### Server health metrics -These are operational metrics to monitor the health of the dstack server. For now, these only include HTTP metrics, but more will be added later. - -=== "Metrics" - | Name | Type | Description | Examples | - |------------------------------------------|-----------|-----------------------------------|--------------| - | `dstack_server_requests_total` | *counter* | Total number of HTTP requests | `100.0` | - | `dstack_server_request_duration_seconds` | *histogram* | HTTP request duration in seconds | `1.0`| - -=== "Labels" - | Name | Type | Description | Examples | - |------------------------|-----------|:--------------|----------------------------------------| - | `method` | *string* | HTTP method | `POST` | - | `endpoint` | *string* | Endpoint path | `/api/project/main/repos/get` | - | `http_status` | *string* | HTTP status code | `200` | - | `project_name` | *string?* | Project name | `main` | +To monitor the health of the `dstack` server itself (HTTP requests, DB queries, process +metrics, and more), use the OpenTelemetry integration — see +[observability](../guides/server-deployment.md#observability). diff --git a/mkdocs/docs/guides/server-deployment.md b/mkdocs/docs/guides/server-deployment.md index d20e9d37e0..caf85f9da7 100644 --- a/mkdocs/docs/guides/server-deployment.md +++ b/mkdocs/docs/guides/server-deployment.md @@ -444,11 +444,20 @@ $ DSTACK_OTEL_TRACES_ENABLED=1 \ - `DSTACK_OTEL_TRACES_BACKGROUND_SAMPLE_RATE` – The head sampling rate for background task traces. ??? info "Metrics exporters" - By default, if the [Prometheus `/metrics` endpoint](../concepts/metrics.md) is enabled via - `DSTACK_ENABLE_PROMETHEUS_METRICS`, OpenTelemetry metrics are exposed there alongside the built-in - `dstack` metrics; otherwise they are pushed via OTLP. Set `DSTACK_OTEL_METRICS_EXPORTERS` - to a comma-separated list of `prometheus` and/or `otlp` to override, e.g. force OTLP push - even when `/metrics` is enabled. + By default, metrics are pushed via OTLP like traces and logs. Set + `DSTACK_OTEL_METRICS_EXPORTERS=prometheus` to instead expose them on the + [Prometheus `/metrics` endpoint](../concepts/metrics.md) alongside the built-in `dstack` + metrics (a comma-separated list enables both exporters at once). + + If you run multiple server replicas, only use the `prometheus` exporter if Prometheus + scrapes each replica directly. Scraping through a load balancer interleaves the + replicas' counters into the same series and silently corrupts all rates. + + When pushed metrics end up in Prometheus (e.g. via an OTel Collector), the sample + cadence is set by `OTEL_METRIC_EXPORT_INTERVAL` (milliseconds; defaults to `60000`). + If it's much larger than Prometheus' configured `timeInterval` (defaults to 15s), + `rate()` queries may be empty. Set e.g. `OTEL_METRIC_EXPORT_INTERVAL=30000`, + and make sure the datasource's `timeInterval` reflects the cadence you choose. ## Encryption diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index a8405edd63..c547765cd0 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -130,7 +130,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_OTEL_TRACES_BACKGROUND_SAMPLE_RATE`{ #DSTACK_OTEL_TRACES_BACKGROUND_SAMPLE_RATE } – The head sampling rate for background task traces. Defaults to `1.0`. - `DSTACK_OTEL_LOGS_ENABLED`{ #DSTACK_OTEL_LOGS_ENABLED } – Enables server log export via OTLP if set to any value. Requires the `otel` extra. - `DSTACK_OTEL_METRICS_ENABLED`{ #DSTACK_OTEL_METRICS_ENABLED } – Enables OpenTelemetry metrics if set to any value. Requires the `otel` extra. -- `DSTACK_OTEL_METRICS_EXPORTERS`{ #DSTACK_OTEL_METRICS_EXPORTERS } – A comma-separated list of OpenTelemetry metrics exporters: `prometheus` (expose on the `/metrics` endpoint) and/or `otlp` (push via OTLP). Defaults to `prometheus` if `DSTACK_ENABLE_PROMETHEUS_METRICS` is set, otherwise `otlp`. +- `DSTACK_OTEL_METRICS_EXPORTERS`{ #DSTACK_OTEL_METRICS_EXPORTERS } – A comma-separated list of OpenTelemetry metrics exporters: `otlp` (push via OTLP) and/or `prometheus` (expose on the `/metrics` endpoint). Defaults to `otlp`. - `DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE`{ #DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE } – Request body size limit for services running with a gateway, in bytes. Defaults to 64 MiB. - `DSTACK_SERVICE_CLIENT_TIMEOUT`{ #DSTACK_SERVICE_CLIENT_TIMEOUT } – Timeout in seconds for HTTP requests sent from the in-server proxy and gateways to service replicas. Defaults to 60. - `DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY`{ #DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY } – Forbids registering new services without a gateway if set to any value. diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 93151b7a8d..6b1c4cff71 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -13,7 +13,6 @@ from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from packaging.version import Version -from prometheus_client import Counter, Histogram from dstack._internal import settings as core_settings from dstack._internal.cli.utils.common import console @@ -51,11 +50,13 @@ users, volumes, ) +from dstack._internal.server.services import prometheus as prometheus_service from dstack._internal.server.services.config import ServerConfigManager from dstack._internal.server.services.gateways import gateway_connections_pool from dstack._internal.server.services.jobs.server_connection import job_server_connections_pool from dstack._internal.server.services.locking import advisory_lock_ctx from dstack._internal.server.services.projects import get_or_create_default_project +from dstack._internal.server.services.prometheus.client_metrics import http_metrics from dstack._internal.server.services.proxy.deps import ServerProxyDependencyInjector from dstack._internal.server.services.proxy.routers import service_proxy from dstack._internal.server.services.runner.pool import instance_connection_pool @@ -83,20 +84,9 @@ logger = get_logger(__name__) -# Server HTTP metrics -REQUESTS_TOTAL = Counter( - "dstack_server_requests_total", - "Total number of HTTP requests", - ["method", "endpoint", "http_status", "project_name"], -) -REQUEST_DURATION = Histogram( - "dstack_server_request_duration_seconds", - "HTTP request duration in seconds", - ["method", "endpoint", "http_status", "project_name"], -) - def create_app() -> FastAPI: + prometheus_service.unregister_default_collectors() app = FastAPI( docs_url="/api/docs", lifespan=lifespan, @@ -361,19 +351,13 @@ def _extract_endpoint_label(request: Request, response: Response) -> str: response: Response = await call_next(request) endpoint_label = _extract_endpoint_label(request, response) - REQUEST_DURATION.labels( + http_metrics.log_request( method=request.method, endpoint=endpoint_label, http_status=response.status_code, project_name=project_name, - ).observe(request.state.process_time) - - REQUESTS_TOTAL.labels( - method=request.method, - endpoint=endpoint_label, - http_status=response.status_code, - project_name=project_name, - ).inc() + duration_seconds=request.state.process_time, + ) return response @app.get("/healthcheck") diff --git a/src/dstack/_internal/server/identity.py b/src/dstack/_internal/server/identity.py new file mode 100644 index 0000000000..085cc57614 --- /dev/null +++ b/src/dstack/_internal/server/identity.py @@ -0,0 +1,7 @@ +import uuid + +SERVER_REPLICA_ID = str(uuid.uuid4()) +""" +Unique ID of this server process, regenerated on every start. +Distinguishes replicas in multi-replica deployments, e.g. in exported telemetry. +""" diff --git a/src/dstack/_internal/server/services/prometheus/__init__.py b/src/dstack/_internal/server/services/prometheus/__init__.py index e69de29bb2..0697d7b7e1 100644 --- a/src/dstack/_internal/server/services/prometheus/__init__.py +++ b/src/dstack/_internal/server/services/prometheus/__init__.py @@ -0,0 +1,20 @@ +from contextlib import suppress + +import prometheus_client + + +def unregister_default_collectors() -> None: + """Removes the collectors prometheus_client registers by default. + + The default collectors (process_*, python_gc_*, python_info) are per-process: + scraping them through a load balancer with multiple server replicas interleaves + the replicas' values into the same series, corrupting them. Process metrics + are available via OpenTelemetry instead, with a per-replica identity. + """ + for collector in ( + prometheus_client.PROCESS_COLLECTOR, + prometheus_client.PLATFORM_COLLECTOR, + prometheus_client.GC_COLLECTOR, + ): + with suppress(KeyError): + prometheus_client.REGISTRY.unregister(collector) diff --git a/src/dstack/_internal/server/services/prometheus/client_metrics.py b/src/dstack/_internal/server/services/prometheus/client_metrics.py index d2c5da9db6..ca6132e503 100644 --- a/src/dstack/_internal/server/services/prometheus/client_metrics.py +++ b/src/dstack/_internal/server/services/prometheus/client_metrics.py @@ -1,3 +1,5 @@ +from typing import Optional + from prometheus_client import Counter, Histogram @@ -53,3 +55,44 @@ def increment_pending_runs(self, project_name: str, run_type: str): run_metrics = RunMetrics() + + +class HTTPMetrics: + """Wrapper class for server HTTP Prometheus metrics. + + Deprecated in favor of the OpenTelemetry HTTP metrics + (`DSTACK_OTEL_METRICS_ENABLED`), which are correct in multi-replica + deployments. Kept for backward compatibility. + """ + + def __init__(self): + self._requests_total = Counter( + "dstack_server_requests_total", + "Total number of HTTP requests", + labelnames=["method", "endpoint", "http_status", "project_name"], + ) + self._request_duration = Histogram( + "dstack_server_request_duration_seconds", + "HTTP request duration in seconds", + labelnames=["method", "endpoint", "http_status", "project_name"], + ) + + def log_request( + self, + method: str, + endpoint: str, + http_status: int, + project_name: Optional[str], + duration_seconds: float, + ): + labels = { + "method": method, + "endpoint": endpoint, + "http_status": http_status, + "project_name": project_name, + } + self._request_duration.labels(**labels).observe(duration_seconds) + self._requests_total.labels(**labels).inc() + + +http_metrics = HTTPMetrics() diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index b3e321dc06..b7be0bf3b2 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -136,9 +136,9 @@ OTEL_METRICS_ENABLED = os.getenv("DSTACK_OTEL_METRICS_ENABLED") is not None """Enables OpenTelemetry metrics. Requires the `otel` extra to be installed.""" OTEL_METRICS_EXPORTERS = os.getenv("DSTACK_OTEL_METRICS_EXPORTERS") -"""A comma-separated list of metrics exporters: `prometheus` (expose via the /metrics endpoint) -and/or `otlp` (push via OTLP, configured by standard `OTEL_*` env vars). -Defaults to `prometheus` if the /metrics endpoint is enabled, otherwise `otlp`. +"""A comma-separated list of metrics exporters: `otlp` (push via OTLP, configured by +standard `OTEL_*` env vars) and/or `prometheus` (expose via the /metrics endpoint). +Defaults to `otlp`. """ DEFAULT_CREDS_DISABLED = os.getenv("DSTACK_DEFAULT_CREDS_DISABLED") is not None diff --git a/src/dstack/_internal/server/utils/otel/utils.py b/src/dstack/_internal/server/utils/otel/utils.py index 006591e7d8..d78385862d 100644 --- a/src/dstack/_internal/server/utils/otel/utils.py +++ b/src/dstack/_internal/server/utils/otel/utils.py @@ -37,6 +37,7 @@ from dstack._internal import settings as core_settings from dstack._internal.server import settings +from dstack._internal.server.identity import SERVER_REPLICA_ID from dstack._internal.server.utils.common import is_background_task_name from dstack._internal.server.utils.logging import AsyncioCancelledErrorFilter from dstack._internal.utils.logging import get_logger @@ -110,10 +111,12 @@ def _instrument(app: FastAPI, engine: AsyncEngine) -> None: def _get_metrics_exporters() -> List[str]: + # OTLP by default, like traces and logs. The prometheus exporter (metrics on + # the /metrics endpoint) is opt-in: scraping /metrics through a load + # balancer with multiple server replicas interleaves the replicas' counters + # into the same series, silently corrupting rates. if settings.OTEL_METRICS_EXPORTERS is not None: return [e.strip() for e in settings.OTEL_METRICS_EXPORTERS.split(",") if e.strip()] - if settings.ENABLE_PROMETHEUS_METRICS: - return ["prometheus"] return ["otlp"] @@ -137,6 +140,9 @@ def _get_resource() -> Resource: return Resource.create( { "service.name": "dstack-server", + # Distinguishes replicas: without it, metrics pushed by multiple + # server replicas collapse into the same series, corrupting counters + "service.instance.id": SERVER_REPLICA_ID, "service.version": core_settings.DSTACK_VERSION or "dev", "deployment.environment.name": settings.SERVER_ENVIRONMENT, } diff --git a/src/tests/_internal/server/utils/test_tracing.py b/src/tests/_internal/server/utils/test_tracing.py index 02c09d1f17..e9168992e1 100644 --- a/src/tests/_internal/server/utils/test_tracing.py +++ b/src/tests/_internal/server/utils/test_tracing.py @@ -148,19 +148,17 @@ def test_does_not_export_otel_sdk_records(self): class TestGetMetricsExporters: @pytest.mark.parametrize( - ("exporters", "prometheus_enabled", "expected"), + ("exporters", "expected"), [ - (None, True, ["prometheus"]), - (None, False, ["otlp"]), - ("otlp", True, ["otlp"]), - ("prometheus, otlp", False, ["prometheus", "otlp"]), + (None, ["otlp"]), + ("prometheus", ["prometheus"]), + ("prometheus, otlp", ["prometheus", "otlp"]), ], ) - def test_returns_expected(self, monkeypatch, exporters, prometheus_enabled, expected): + def test_returns_expected(self, monkeypatch, exporters, expected): from dstack._internal.server import settings monkeypatch.setattr(settings, "OTEL_METRICS_EXPORTERS", exporters) - monkeypatch.setattr(settings, "ENABLE_PROMETHEUS_METRICS", prometheus_enabled) assert _get_metrics_exporters() == expected