diff --git a/.claude/commands/run-challenger-node.md b/.claude/commands/run-challenger-node.md index fbf80345..c3b3a93a 100644 --- a/.claude/commands/run-challenger-node.md +++ b/.claude/commands/run-challenger-node.md @@ -7,6 +7,7 @@ The Challenger verifies operator operations and submits challenges if necessary. 1. Ask the user for the following parameters (skip any already provided as arguments: $ARGUMENTS): - **network**: Which network? `testnet4` or `regtest` - **rpc_addr**: RPC listen address. Default: `127.0.0.1:8906` + - **metrics_addr**: Prometheus listen address. Default: `127.0.0.1:9902` - **p2p_port**: P2P listen port. Default: `8449` (testnet4) or `8450` (regtest) - **db_path**: SQLite database path. Default: `sqlite:$PWD/bitvm-node.db` @@ -32,26 +33,28 @@ The Challenger verifies operator operations and submits challenges if necessary. 4. Start the challenger node: ```bash -./bin/bitvm-noded --rpc-addr --db-path --p2p-port --bootnodes "$BOOTNODES" +./bin/bitvm-noded --rpc-addr --metrics-addr --db-path --p2p-port --bootnodes "$BOOTNODES" ``` To run in the background: ```bash -nohup ./bin/bitvm-noded --rpc-addr --db-path --p2p-port --bootnodes "$BOOTNODES" >challenger_$(date +'%Y%m%d').log 2>&1 & +nohup ./bin/bitvm-noded --rpc-addr --metrics-addr --db-path --p2p-port --bootnodes "$BOOTNODES" >challenger_$(date +'%Y%m%d').log 2>&1 & ``` 5. Verify the node is running: ```bash curl -s http:/// +curl -s http:///metrics ``` -Should return `Hello, World!`. +The first request should return `Hello, World!`; the second should return +Prometheus metrics in OpenMetrics text format. ### Example (testnet4) ```bash cp deployment/testnet4/bitvm-noded/challenge_0/.env.challenge_0 .env # Edit .env to fill in BITVM_SECRET, GOAT_ADDRESS, PEER_KEY -./bin/bitvm-noded --rpc-addr 127.0.0.1:8906 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8449 --bootnodes /ip4/34.215.238.232/tcp/8445/p2p/12D3KooWCrPTAmhFdC5DBGgkxZvJi6iuSeiDWKRL87isrt4iMHXv +./bin/bitvm-noded --rpc-addr 127.0.0.1:8906 --metrics-addr 127.0.0.1:9902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8449 --bootnodes /ip4/34.215.238.232/tcp/8445/p2p/12D3KooWCrPTAmhFdC5DBGgkxZvJi6iuSeiDWKRL87isrt4iMHXv ``` For full deployment documentation, see `deployment/README.md` (section **Challenger**). diff --git a/.claude/commands/run-operator-node.md b/.claude/commands/run-operator-node.md index b82f3c84..90b68bff 100644 --- a/.claude/commands/run-operator-node.md +++ b/.claude/commands/run-operator-node.md @@ -7,6 +7,7 @@ The Operator manages bridge operations, kickoff processing, and pegout (Gateway. 1. Ask the user for the following parameters (skip any already provided as arguments: $ARGUMENTS): - **network**: Which network? `testnet4` or `regtest` - **rpc_addr**: RPC listen address. Default: `127.0.0.1:8902` + - **metrics_addr**: Prometheus listen address. Default: `127.0.0.1:9903` - **p2p_port**: P2P listen port. Default: `8445` (testnet4) or `8446` (regtest) - **db_path**: SQLite database path. Default: `sqlite:$PWD/bitvm-node.db` @@ -35,26 +36,28 @@ The Operator manages bridge operations, kickoff processing, and pegout (Gateway. 4. Start the operator node: ```bash -./bin/bitvm-noded --rpc-addr --db-path --p2p-port --bootnodes "$BOOTNODES" +./bin/bitvm-noded --rpc-addr --metrics-addr --db-path --p2p-port --bootnodes "$BOOTNODES" ``` To run in the background: ```bash -nohup ./bin/bitvm-noded --rpc-addr --db-path --p2p-port --bootnodes "$BOOTNODES" >operator_$(date +'%Y%m%d').log 2>&1 & +nohup ./bin/bitvm-noded --rpc-addr --metrics-addr --db-path --p2p-port --bootnodes "$BOOTNODES" >operator_$(date +'%Y%m%d').log 2>&1 & ``` 5. Verify the node is running: ```bash curl -s http:/// +curl -s http:///metrics ``` -Should return `Hello, World!`. +The first request should return `Hello, World!`; the second should return +Prometheus metrics in OpenMetrics text format. ### Example (testnet4) ```bash cp deployment/testnet4/bitvm-noded/operator_0/.env.operator_0 .env # Edit .env to fill in BITVM_SECRET, GOAT_ADDRESS, PEER_KEY, GOAT_PRIVATE_KEY -./bin/bitvm-noded --rpc-addr 127.0.0.1:8902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 --bootnodes /ip4/34.215.238.232/tcp/8445/p2p/12D3KooWCrPTAmhFdC5DBGgkxZvJi6iuSeiDWKRL87isrt4iMHXv +./bin/bitvm-noded --rpc-addr 127.0.0.1:8902 --metrics-addr 127.0.0.1:9903 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 --bootnodes /ip4/34.215.238.232/tcp/8445/p2p/12D3KooWCrPTAmhFdC5DBGgkxZvJi6iuSeiDWKRL87isrt4iMHXv ``` For full deployment documentation, see `deployment/README.md` (section **Operator**). diff --git a/crates/bitcoin-light-client-circuit/src/lib.rs b/crates/bitcoin-light-client-circuit/src/lib.rs index 7f58c96f..1882d06d 100644 --- a/crates/bitcoin-light-client-circuit/src/lib.rs +++ b/crates/bitcoin-light-client-circuit/src/lib.rs @@ -33,6 +33,12 @@ pub const VK_HASH_SIZE: usize = 66; pub const TOTAL_WORK_SIZE: usize = 32; pub const CONSENSUS_BLOCK_HEIGHT_SIZE: usize = 4; +// 0xc3d8382a8efed79d564f8ec6da4dd00a4e77572d1e02f5f81dda441a36fdc9d8 +pub const ROOT_COMMIT_PROGRAM_ID: verifier::ProgramId = [ + 195, 216, 56, 42, 142, 254, 215, 157, 86, 79, 142, 198, 218, 77, 208, 10, 78, 119, 87, 45, 30, + 2, 245, 248, 29, 218, 68, 26, 54, 253, 201, 216, +]; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WatchtowerPublicOutputs { pub total_work: [u8; TOTAL_WORK_SIZE], @@ -74,6 +80,11 @@ fn checked_history_root( } /// Verifies that the latest Bitcoin commitment authorizes every supplied circuit proof. +/// +/// `root_commit_program_id` is the pinned anchor; it is the only identity here that does not +/// come from the proofs being checked, so it is asserted before anything reads the +/// Commit-chain output. See [`ROOT_COMMIT_PROGRAM_ID`]. +#[allow(clippy::too_many_arguments)] fn verify_commitment_authorization( commit_chain_output: &CommitChainCircuitOutput, header_chain_output: &BlockHeaderCircuitOutput, @@ -81,8 +92,13 @@ fn verify_commitment_authorization( header_program_id: verifier::ProgramId, state_program_id: verifier::ProgramId, commit_program_id: verifier::ProgramId, + root_commit_program_id: verifier::ProgramId, sequencer_set_hash: [u8; 32], ) -> AuthorizedProgramIds { + assert_eq!( + commit_program_id, root_commit_program_id, + "commit proof program id does not match the pinned root anchor" + ); let authorized = commit_chain_output.chain_state.authorized_program_ids; authorized.validate().expect("invalid authorized ProgramIds"); check_program_id(commit_program_id, commit_chain_output.self_program_id, authorized.commit); @@ -215,6 +231,7 @@ pub fn watch_longest_chain( header_program_id, state_program_id, commit_program_id, + ROOT_COMMIT_PROGRAM_ID, sequencer_set_hash, ); } else { @@ -382,6 +399,7 @@ pub fn propose_longest_chain( header_program_id, state_program_id, commit_program_id, + ROOT_COMMIT_PROGRAM_ID, sequencer_set_hash, ) } else { diff --git a/deployment/README.md b/deployment/README.md index 2126dd23..62b45351 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -69,10 +69,29 @@ Configure the `.env` file for the specific role and run the node. **Common Arguments:** - `--rpc-addr `: Address for the RPC server (default: `0.0.0.0:8080`) +- `--metrics-addr `: Address for the dedicated Prometheus listener (disabled when omitted) +- `--metrics-path `: Path served by the metrics listener (default: `/metrics`) - `--db-path `: Path to the SQLite database (default: `sqlite:/tmp/bitvm-node.db`) - `--p2p-port `: Port for P2P communication (default: `0`) - `--bootnodes `: Bootnodes to connect to +Bind metrics to a private interface and use a unique port for every node running +on the same host. The business RPC listener does not serve metrics, so +`GET /metrics` on `--rpc-addr` returns `404`. For remote Prometheus servers, +expose the dedicated listener through a private network, proxy, or access-control +layer instead of binding it directly to a public interface. + +The provided Docker Compose deployment enables the listener on port `9108` +inside each container and publishes it only on the host loopback interface: + +| Service | Host metrics endpoint | +|---------|-----------------------| +| `committee_0` | `http://127.0.0.1:9900/metrics` | +| `committee_1` | `http://127.0.0.1:9901/metrics` | +| `verifier_0` | `http://127.0.0.1:9902/metrics` | +| `operator_0` | `http://127.0.0.1:9903/metrics` | +| `watchtower_0` | `http://127.0.0.1:9904/metrics` | + ### Committee The Committee is responsible for signing presign transactions. @@ -113,6 +132,7 @@ GOAT_SWAP_EVENT_FILTER_FROM=9368978 ```bash ./target/release/bitvm-noded \ --rpc-addr 0.0.0.0:9100 \ + --metrics-addr 127.0.0.1:9108 \ --p2p-port 8443 \ --db-path ./committee.db ``` @@ -148,6 +168,7 @@ BOOTNODES=... # empty if this is the first node ```bash ./target/release/bitvm-noded \ --rpc-addr 0.0.0.0:9100 \ + --metrics-addr 127.0.0.1:9108 \ --p2p-port 8443 \ --db-path ./operator.db ``` @@ -183,6 +204,7 @@ BOOTNODES=... # empty if this is the first node ```bash ./target/release/bitvm-noded \ --rpc-addr 0.0.0.0:9100 \ + --metrics-addr 127.0.0.1:9108 \ --p2p-port 8443 \ --db-path ./challenger.db ``` @@ -217,6 +239,7 @@ BOOTNODES=... # empty if this is the first node ```bash ./target/release/bitvm-noded \ --rpc-addr 0.0.0.0:9100 \ + --metrics-addr 127.0.0.1:9108 \ --p2p-port 8443 \ --db-path ./watchtower.db ``` diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 3ec15d30..e2333c25 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -8,12 +8,15 @@ services: volumes: - ./committee_0:/var/data - ./committee_0/.env:/app/.env - ports: - - "8900:9100" - - "8443:8443" env_file: - ./committee_0/.env + environment: + METRICS_ADDR: "0.0.0.0:9108" restart: unless-stopped + ports: + - "8900:9100" + - "8443:8443" + - "127.0.0.1:9900:9108" committee_1: image: ghcr.io/goatnetwork/bitvm-node:${TAG:-0.1.0} @@ -24,8 +27,11 @@ services: ports: - "8901:9100" - "8444:8443" + - "127.0.0.1:9901:9108" env_file: - ./committee_1/.env + environment: + METRICS_ADDR: "0.0.0.0:9108" restart: unless-stopped depends_on: - committee_0 @@ -39,8 +45,11 @@ services: ports: - "8902:9100" - "8445:8443" + - "127.0.0.1:9902:9108" env_file: - ./verifier_0/.env + environment: + METRICS_ADDR: "0.0.0.0:9108" restart: unless-stopped depends_on: - committee_0 @@ -54,8 +63,11 @@ services: ports: - "8903:9100" - "8456:8443" + - "127.0.0.1:9903:9108" env_file: - ./operator_0/.env + environment: + METRICS_ADDR: "0.0.0.0:9108" restart: unless-stopped depends_on: - committee_0 @@ -69,8 +81,11 @@ services: ports: - "8904:9100" - "8457:8443" + - "127.0.0.1:9904:9108" env_file: - ./watchtower_0/.env + environment: + METRICS_ADDR: "0.0.0.0:9108" restart: unless-stopped depends_on: - - committee_0 \ No newline at end of file + - committee_0 diff --git a/deployment/regtest/bitvm-noded/committee_0/run.sh b/deployment/regtest/bitvm-noded/committee_0/run.sh index 0ce409bb..a074600c 100644 --- a/deployment/regtest/bitvm-noded/committee_0/run.sh +++ b/deployment/regtest/bitvm-noded/committee_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 0.0.0.0:8900 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8444 >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 0.0.0.0:8900 --metrics-addr 127.0.0.1:9900 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8444 >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/regtest/bitvm-noded/committee_1/run.sh b/deployment/regtest/bitvm-noded/committee_1/run.sh index 73c64c84..5dd31c89 100644 --- a/deployment/regtest/bitvm-noded/committee_1/run.sh +++ b/deployment/regtest/bitvm-noded/committee_1/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8901 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8901 --metrics-addr 127.0.0.1:9901 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/regtest/bitvm-noded/operator_0/run.sh b/deployment/regtest/bitvm-noded/operator_0/run.sh index fe8e5c7e..e0c2a412 100644 --- a/deployment/regtest/bitvm-noded/operator_0/run.sh +++ b/deployment/regtest/bitvm-noded/operator_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8446 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8902 --metrics-addr 127.0.0.1:9903 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8446 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/regtest/bitvm-noded/verifier_0/run.sh b/deployment/regtest/bitvm-noded/verifier_0/run.sh index cb09ad86..e0fc8a41 100644 --- a/deployment/regtest/bitvm-noded/verifier_0/run.sh +++ b/deployment/regtest/bitvm-noded/verifier_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8906 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8450 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8906 --metrics-addr 127.0.0.1:9902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8450 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/regtest/bitvm-noded/watchtower_0/run.sh b/deployment/regtest/bitvm-noded/watchtower_0/run.sh index 740b5947..3cbc2b73 100644 --- a/deployment/regtest/bitvm-noded/watchtower_0/run.sh +++ b/deployment/regtest/bitvm-noded/watchtower_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8904 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8448 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8904 --metrics-addr 127.0.0.1:9904 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8448 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/testnet4/bitvm-noded/committee_0/run.sh b/deployment/testnet4/bitvm-noded/committee_0/run.sh index eea944bf..cbac102c 100644 --- a/deployment/testnet4/bitvm-noded/committee_0/run.sh +++ b/deployment/testnet4/bitvm-noded/committee_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 >$PWD/$(date +'%Y%m%d').log 2>&1 & +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8902 --metrics-addr 127.0.0.1:9900 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/testnet4/bitvm-noded/committee_1/run.sh b/deployment/testnet4/bitvm-noded/committee_1/run.sh index b5ac2a6f..15356a59 100644 --- a/deployment/testnet4/bitvm-noded/committee_1/run.sh +++ b/deployment/testnet4/bitvm-noded/committee_1/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8901 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8444 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8901 --metrics-addr 127.0.0.1:9901 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8444 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/testnet4/bitvm-noded/operator_0/run.sh b/deployment/testnet4/bitvm-noded/operator_0/run.sh index 2d7c3339..85e6f32e 100644 --- a/deployment/testnet4/bitvm-noded/operator_0/run.sh +++ b/deployment/testnet4/bitvm-noded/operator_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8902 --metrics-addr 127.0.0.1:9903 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8445 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/testnet4/bitvm-noded/verifier_0/run.sh b/deployment/testnet4/bitvm-noded/verifier_0/run.sh index 2d0b213c..e977a155 100644 --- a/deployment/testnet4/bitvm-noded/verifier_0/run.sh +++ b/deployment/testnet4/bitvm-noded/verifier_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8906 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8449 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8906 --metrics-addr 127.0.0.1:9902 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8449 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/deployment/testnet4/bitvm-noded/watchtower_0/run.sh b/deployment/testnet4/bitvm-noded/watchtower_0/run.sh index a83ceea0..03c9dec1 100644 --- a/deployment/testnet4/bitvm-noded/watchtower_0/run.sh +++ b/deployment/testnet4/bitvm-noded/watchtower_0/run.sh @@ -1 +1 @@ -nohup ../bitvm-noded --rpc-addr 127.0.0.1:8904 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8447 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & \ No newline at end of file +nohup ../bitvm-noded --rpc-addr 127.0.0.1:8904 --metrics-addr 127.0.0.1:9904 --db-path sqlite:$PWD/bitvm-node.db --p2p-port 8447 --bootnodes $bootnode_urls >$PWD/$(date +'%Y%m%d').log 2>&1 & diff --git a/node/README.md b/node/README.md index e1c744a1..8cff4628 100644 --- a/node/README.md +++ b/node/README.md @@ -814,6 +814,7 @@ bitvm-noded key funding-address ```bash bitvm-noded \ --rpc-addr 0.0.0.0:8080 \ + --metrics-addr 127.0.0.1:9108 \ --db-path ./node.db \ --p2p-port 4001 \ --bootnodes /ip4/x.x.x.x/tcp/4001/p2p/ @@ -850,7 +851,8 @@ real graph raw data in the database. | `--db-path` | SQLite database path | `sqlite:/tmp/bitvm-node.db` | | `--p2p-port` | P2P listen port | `0` (random) | | `--bootnodes` | Bootstrap node addresses | - | -| `--metrics-path` | Prometheus metrics endpoint | `/metrics` | +| `--metrics-addr` | Dedicated Prometheus listener address | Disabled | +| `--metrics-path` | Path served by the dedicated metrics listener | `/metrics` | | `--enable-kademlia` | Enable Kademlia DHT | `true` | --- @@ -899,6 +901,10 @@ Relayer nodes should: ## RPC API +Prometheus metrics are not served by the business RPC listener. Set +`--metrics-addr` to a private, process-unique address and scrape +`http:///metrics`; `GET /metrics` on `--rpc-addr` returns `404`. + ### Endpoints | Endpoint | Method | Description | @@ -918,7 +924,7 @@ Relayer nodes should: | `/v1/swaps` | GET | List swap bridge-out escrows | | `/v1/swaps/:escrow_hash` | GET | Get swap escrow details | | `/challenge` | POST | Submit challenge | -| `/metrics` | GET | Prometheus metrics | +| `/metrics` | GET | Prometheus metrics (dedicated metrics listener only) | --- diff --git a/node/entrypoint.sh b/node/entrypoint.sh index 4e4d6b66..b165f698 100755 --- a/node/entrypoint.sh +++ b/node/entrypoint.sh @@ -1,8 +1,13 @@ #!/bin/bash -bn="" -if [ -n "$BOOTNODES" ]; then - bn="--bootnodes $BOOTNODES" +args=(--rpc-addr 0.0.0.0:9100 --db-path /var/data/bitvm-node-0.db --p2p-port 8443) + +if [ -n "${BOOTNODES:-}" ]; then + args+=(--bootnodes "$BOOTNODES") +fi + +if [ -n "${METRICS_ADDR:-}" ]; then + args+=(--metrics-addr "$METRICS_ADDR") fi -bitvm-noded --rpc-addr 0.0.0.0:9100 --db-path /var/data/bitvm-node-0.db --p2p-port 8443 $bn +exec bitvm-noded "${args[@]}" diff --git a/proof-builder-rpc/README.md b/proof-builder-rpc/README.md index b2c1eb0f..b7f43bc1 100644 --- a/proof-builder-rpc/README.md +++ b/proof-builder-rpc/README.md @@ -52,6 +52,23 @@ To enable multiple Gateways without interrupting proof polling, first upgrade Pr single Gateway, then upgrade all Nodes to send `gateway_address`, and finally configure the comma-separated Gateway list. +## Metrics + +Prometheus metrics use a dedicated listener and are disabled unless `--metrics-addr` is set. The +business RPC listener does not serve metrics, so `GET /metrics` on `--rpc-addr` returns `404`. + +```bash +proof-builder-rpc \ + --rpc-addr 0.0.0.0:7777 \ + --metrics-addr 10.42.0.8:9109 \ + --config proof-builder.toml +``` + +The dedicated listener serves only `/metrics`. Bind it to a private interface and allow only the +Prometheus source network to reach it. The listener does not provide TLS or authentication and +should not be exposed directly to the public internet. An unavailable metrics port causes startup +to fail. + ## Failure recovery Long-running proof tasks (stored in the `long_running_task_proof` table) — such as header-chain, commit-chain, and state-chain proofs — can be recovered from the database. Recovery notes: diff --git a/proof-builder-rpc/src/api/mod.rs b/proof-builder-rpc/src/api/mod.rs index 1758e4d2..262116d1 100644 --- a/proof-builder-rpc/src/api/mod.rs +++ b/proof-builder-rpc/src/api/mod.rs @@ -12,6 +12,7 @@ use crate::api::proof_handler::{ post_watchtower_proof_task, update_operator_proof_task_timeout, update_watchtower_proof_task_timeout, }; +use anyhow::Context; use axum::http::{Method, StatusCode}; use axum::routing::{get, post}; use axum::{Router, middleware}; @@ -23,15 +24,15 @@ use tower_http::cors::{Any, CorsLayer}; pub(crate) use auth::{AuthorizationChain, AuthorizationChains}; -struct ApiState { +pub(crate) struct ApiState { pub local_db: LocalDB, pub metrics_state: ApiMetricsState, - pub auth: RequestAuthorizer, + auth: RequestAuthorizer, } impl ApiState { /// Creates shared API state from the database, metrics, and live authorization chain. - fn new( + pub(crate) fn new( local_db: LocalDB, metrics_state: ApiMetricsState, authorization_chains: AuthorizationChains, @@ -43,14 +44,13 @@ impl ApiState { }) } } -pub(crate) async fn serve( + +/// Serves the business RPC routes at `addr` with shared state until cancellation. +pub(crate) async fn serve_with_app_state( addr: String, - local_db: LocalDB, - metrics_state: ApiMetricsState, - authorization_chains: AuthorizationChains, + api_state: Arc, cancellation_token: CancellationToken, ) -> anyhow::Result { - let api_state = ApiState::new(local_db, metrics_state, authorization_chains); let instrumented_routes = Router::new() .route(routes::ROOT, get(root)) .route(routes::v1::PROOFS_CHAIN_PROOFS_DESC, get(get_chain_proof_task_desc)) @@ -64,9 +64,7 @@ pub(crate) async fn serve( .route(routes::v1::PROOFS_OPERATOR_PROOF_DESC, get(get_operator_proof_task_desc)) .fallback(|| async { StatusCode::NOT_FOUND }) .layer(middleware::from_fn_with_state(api_state.clone(), metrics_middleware)); - let server = Router::new() - .route(routes::METRICS, get(metrics_handler)) - .merge(instrumented_routes) + let server = instrumented_routes .layer(CorsLayer::new().allow_headers(Any).allow_origin(Any).allow_methods(vec![ Method::GET, Method::POST, @@ -93,6 +91,45 @@ pub(crate) async fn serve( } } } + +/// Binds the dedicated Prometheus metrics listener before background tasks start. +pub(crate) async fn bind_metrics_listener(addr: &str) -> anyhow::Result { + TcpListener::bind(addr) + .await + .with_context(|| format!("failed to bind metrics listener to {addr}")) +} + +/// Serves only `/metrics` from the pre-bound listener with the shared API state. +pub(crate) async fn serve_metrics( + listener: TcpListener, + api_state: Arc, + cancellation_token: CancellationToken, +) -> anyhow::Result { + let router = Router::new().route(routes::METRICS, get(metrics_handler)).with_state(api_state); + let listening_addr = + listener.local_addr().context("failed to determine metrics listener address")?; + tracing::info!( + address = %listening_addr, + path = routes::METRICS, + "Metrics listener started" + ); + + tokio::select! { + result = axum::serve(listener, router) => { + match result { + Ok(_) => Ok("Metrics server finished normally".to_string()), + Err(e) => { + tracing::error!("Metrics server error: {}", e); + Err(anyhow::anyhow!("Metrics server error: {e}")) + } + } + } + _ = cancellation_token.cancelled() => { + tracing::info!("Metrics service received shutdown signal"); + Ok("metrics_shutdown".to_string()) + } + } +} async fn root() -> &'static str { "Hello, World!" } @@ -178,32 +215,59 @@ mod tests { std::collections::HashMap::from([(gateway, chain)]) } + async fn spawn_metrics_listener( + api_state: Arc, + cancellation_token: CancellationToken, + ) -> anyhow::Result<(String, tokio::task::JoinHandle>)> { + let listener = bind_metrics_listener("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + let handle = tokio::spawn(serve_metrics(listener, api_state, cancellation_token)); + Ok((addr, handle)) + } + #[tokio::test] - async fn metrics_use_route_templates_and_exclude_scrapes() -> anyhow::Result<()> { + async fn metrics_are_served_only_by_the_dedicated_listener() -> anyhow::Result<()> { let addr = available_addr(); let cancellation_token = CancellationToken::new(); - let server_token = cancellation_token.clone(); - let server = tokio::spawn(serve( - addr.clone(), + let api_state = ApiState::new( store::create_local_db("sqlite::memory:").await, ApiMetricsState::new(), authorization_chains(gateway(1), Arc::new(TestAuthorizationChain::default())), - server_token, + ); + let server = tokio::spawn(serve_with_app_state( + addr.clone(), + api_state.clone(), + cancellation_token.clone(), )); + let (metrics_addr, metrics_server) = + spawn_metrics_listener(api_state, cancellation_token.clone()).await?; tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert!(get(&addr, "/").await?.starts_with("HTTP/1.1 200")); assert!(get(&addr, "/missing").await?.starts_with("HTTP/1.1 404")); - let first_scrape = get(&addr, "/metrics").await?; - let second_scrape = get(&addr, "/metrics").await?; + assert!(get(&addr, "/metrics").await?.starts_with("HTTP/1.1 404")); + assert!(get(&metrics_addr, "/").await?.starts_with("HTTP/1.1 404")); + assert!( + get(&metrics_addr, routes::v1::PROOFS_CHAIN_PROOFS_DESC) + .await? + .starts_with("HTTP/1.1 404") + ); + let first_scrape = get(&metrics_addr, "/metrics").await?; + let second_scrape = get(&metrics_addr, "/metrics").await?; + assert!(first_scrape.starts_with("HTTP/1.1 200")); + assert!( + first_scrape + .to_ascii_lowercase() + .contains("content-type: application/openmetrics-text;charset=utf-8;version=1.0.0") + ); assert!( first_scrape .contains("http_requests_total{method=\"GET\",route=\"/\",status=\"200\"} 1") ); assert!( first_scrape.contains( - "http_requests_total{method=\"GET\",route=\"unmatched\",status=\"404\"} 1" + "http_requests_total{method=\"GET\",route=\"unmatched\",status=\"404\"} 2" ) ); assert!(first_scrape.contains("http_requests_in_flight 0")); @@ -212,6 +276,19 @@ mod tests { cancellation_token.cancel(); server.await??; + assert_eq!(metrics_server.await??, "metrics_shutdown"); + Ok(()) + } + + #[tokio::test] + async fn metrics_listener_reports_bind_conflicts() -> anyhow::Result<()> { + let occupied = std::net::TcpListener::bind("127.0.0.1:0")?; + let addr = occupied.local_addr()?.to_string(); + let error = bind_metrics_listener(&addr).await.unwrap_err(); + assert!( + error.to_string().contains(&format!("failed to bind metrics listener to {addr}")), + "unexpected error: {error}" + ); Ok(()) } @@ -233,14 +310,13 @@ mod tests { let gateway_address = gateway(1); let addr = available_addr(); let cancellation_token = CancellationToken::new(); - let server_token = cancellation_token.clone(); - let server = tokio::spawn(serve( - addr.clone(), + let api_state = ApiState::new( store::create_local_db("sqlite::memory:").await, ApiMetricsState::new(), authorization_chains(gateway_address, authorization_chain), - server_token, - )); + ); + let server = + tokio::spawn(serve_with_app_state(addr.clone(), api_state, cancellation_token.clone())); tokio::time::sleep(std::time::Duration::from_millis(100)).await; let operator_submit = OperatorProofRequest { @@ -418,14 +494,18 @@ mod tests { std::collections::HashMap::from([(gateway_a, chain_a), (gateway_b, chain_b)]); let addr = available_addr(); let cancellation_token = CancellationToken::new(); - let server_token = cancellation_token.clone(); - let server = tokio::spawn(serve( - addr.clone(), + let api_state = ApiState::new( store::create_local_db("sqlite::memory:").await, ApiMetricsState::new(), authorization_chains, - server_token, + ); + let server = tokio::spawn(serve_with_app_state( + addr.clone(), + api_state.clone(), + cancellation_token.clone(), )); + let (metrics_addr, metrics_server) = + spawn_metrics_listener(api_state, cancellation_token.clone()).await?; tokio::time::sleep(std::time::Duration::from_millis(100)).await; for (gateway_address, expected_status) in [ @@ -453,12 +533,13 @@ mod tests { assert!(response.starts_with(&format!("HTTP/1.1 {expected_status}"))); } - let metrics = get(&addr, "/metrics").await?; + let metrics = get(&metrics_addr, "/metrics").await?; assert!(metrics.contains("operation=\"operator_timeout\",result=\"invalid\"")); assert!(metrics.contains("operation=\"operator_timeout\",result=\"unauthorized\"")); cancellation_token.cancel(); server.await??; + metrics_server.await??; Ok(()) } @@ -471,14 +552,18 @@ mod tests { let gateway_address = gateway(1); let addr = available_addr(); let cancellation_token = CancellationToken::new(); - let server_token = cancellation_token.clone(); - let server = tokio::spawn(serve( - addr.clone(), + let api_state = ApiState::new( store::create_local_db("sqlite::memory:").await, ApiMetricsState::new(), authorization_chains(gateway_address, authorization_chain), - server_token, + ); + let server = tokio::spawn(serve_with_app_state( + addr.clone(), + api_state.clone(), + cancellation_token.clone(), )); + let (metrics_addr, metrics_server) = + spawn_metrics_listener(api_state, cancellation_token.clone()).await?; tokio::time::sleep(std::time::Duration::from_millis(100)).await; let request = OperatorProofTimeoutUpdateRequest { @@ -498,11 +583,12 @@ mod tests { post(&addr, routes::v1::PROOFS_OPERATOR_PROOF_TIMEOUT, &body, Some(&auth)).await?; assert!(response.starts_with("HTTP/1.1 503")); - let metrics = get(&addr, "/metrics").await?; + let metrics = get(&metrics_addr, "/metrics").await?; assert!(metrics.contains("operation=\"operator_timeout\",result=\"unavailable\"")); cancellation_token.cancel(); server.await??; + metrics_server.await??; Ok(()) } } diff --git a/proof-builder-rpc/src/main.rs b/proof-builder-rpc/src/main.rs index e76972d7..0e7fd814 100644 --- a/proof-builder-rpc/src/main.rs +++ b/proof-builder-rpc/src/main.rs @@ -27,6 +27,12 @@ struct Opts { #[arg(long, default_value = "0.0.0.0:7777")] pub rpc_addr: String, + /// Dedicated Prometheus metrics listener address. + /// + /// Unset disables metrics; the business RPC never serves them. + #[arg(long)] + pub metrics_addr: Option, + /// Local Sqlite database file path #[arg(long, env, default_value = "sqlite:/tmp/bitvm-node.db")] pub database_url: String, @@ -44,28 +50,29 @@ async fn main() -> anyhow::Result<()> { println!("proof builder config: {:?}", cfg); let _ = tracing_subscriber::fmt().with_env_filter(EnvFilter::from_default_env()).try_init(); + let metrics_listener = match opt.metrics_addr.as_deref() { + Some(metrics_addr) => Some(api::bind_metrics_listener(metrics_addr).await?), + None => None, + }; let authorization_chains = goat_clients_from_env().await?; // Create cancellation token for graceful shutdown let cancellation_token = CancellationToken::new(); info!("load db {}", opt.database_url); let local_db = store::create_local_db(&opt.database_url).await; let metrics_state = ApiMetricsState::new(); - let local_db_clone1 = local_db.clone(); - let api_metrics_state = metrics_state.clone(); + let api_state = + api::ApiState::new(local_db.clone(), metrics_state.clone(), authorization_chains); let mut task_handles: Vec>> = vec![]; let cancel_token_clone = cancellation_token.clone(); let opt_rpc_addr = opt.rpc_addr.clone(); - info!("start api server"); + let rpc_api_state = api_state.clone(); + info!( + rpc_addr = %opt.rpc_addr, + metrics_addr = opt.metrics_addr.as_deref().unwrap_or("disabled"), + "start api server" + ); task_handles.push(tokio::spawn(async move { - match api::serve( - opt_rpc_addr, - local_db_clone1, - api_metrics_state, - authorization_chains, - cancel_token_clone, - ) - .await - { + match api::serve_with_app_state(opt_rpc_addr, rpc_api_state, cancel_token_clone).await { Ok(tag) => Ok(tag), Err(e) => { tracing::error!("RPC service error: {}", e); @@ -73,6 +80,18 @@ async fn main() -> anyhow::Result<()> { } } })); + if let Some(listener) = metrics_listener { + let cancel_token_clone = cancellation_token.clone(); + task_handles.push(tokio::spawn(async move { + match api::serve_metrics(listener, api_state, cancel_token_clone).await { + Ok(tag) => Ok(tag), + Err(e) => { + tracing::error!("Metrics service error: {}", e); + Err("metrics_error".to_string()) + } + } + })); + } if is_start_generate_proof_tasks(&cfg) { info!("start generate proof tasks"); let cancel_token_clone = cancellation_token.clone();