Add support for connecting to enforcer via TLS, avoid crashing if health service is not available

app/Cargo.toml

@@ -32,7 +32,7 @@ thunder_app_cli = { path = "../cli" }
 thunder_app_rpc_api = { path = "../rpc-api" }
 tiny-bip39 = { workspace = true }
 tokio-util = { workspace = true, features = ["rt"] }
-tonic = { workspace = true }
+tonic = { workspace = true, features = ["tls-ring", "tls-webpki-roots"] }
 tonic-health = { workspace = true }
 # needs to line up with jsonrpsee tower version...
 tower = { workspace = true }

app/app.rs

@@ -19,7 +19,9 @@ use thunder::{
         proto::mainchain::{
             self,
             generated::{
-                block_producer_service_server, mining_service_server,
+                GetChainTipRequest, block_producer_service_server,
+                mining_service_server,
+                validator_service_client::ValidatorServiceClient,
                 validator_service_server, wallet_service_server,
             },
         },
@@ -41,6 +43,8 @@ pub enum Error {
     CusfMainchain(#[from] thunder::types::proto::Error),
     #[error("io error")]
     Io(#[from] std::io::Error),
+    #[error("Invalid mainchain gRPC TLS config")]
+    MainchainTls(#[source] tonic::transport::Error),
     #[error("miner error")]
     Miner(#[from] miner::Error),
     #[error("node error")]
@@ -66,7 +70,10 @@ impl From<node::Error> for Error {
     }
 }
 
-fn update_wallet(node: &Node, wallet: &Wallet) -> Result<(), Error> {
+fn update_wallet(
+    node: &Node<MainchainTransport>,
+    wallet: &Wallet,
+) -> Result<(), Error> {
     tracing::trace!("starting wallet update");
     let addresses = wallet.get_addresses()?;
     let utxos = node.get_utxos_by_addresses(&addresses)?;
@@ -85,7 +92,7 @@ fn update_wallet(node: &Node, wallet: &Wallet) -> Result<(), Error> {
 
 /// Update utxos & wallet
 fn update(
-    node: &Node,
+    node: &Node<MainchainTransport>,
     utxos: &mut HashMap<OutPoint, Output>,
     wallet: &Wallet,
 ) -> Result<(), Error> {
@@ -117,6 +124,8 @@ pub struct Config {
     pub add_peers: HashSet<thunder::types::net::PeerAddress>,
     pub datadir: PathBuf,
     pub mainchain_grpc_url: url::Url,
+    /// Extra CA certificate (PEM) to trust for an `https` mainchain URL
+    pub mainchain_grpc_ca_cert: Option<PathBuf>,
     pub mnemonic_seed_phrase_path: Option<PathBuf>,
     pub net_addr: SocketAddr,
     pub network: thunder::types::Network,
@@ -125,11 +134,15 @@ pub struct Config {
     pub wallet_dir: PathBuf,
 }
 
+/// Connection to the mainchain node
+pub type MainchainTransport =
+    crate::path_prefix::PathPrefix<tonic::transport::Channel>;
+
 #[derive(Clone)]
 pub struct App {
-    pub node: Arc<Node>,
+    pub node: Arc<Node<MainchainTransport>>,
     pub wallet: Wallet,
-    pub miner: Option<Arc<TokioRwLock<Miner>>>,
+    pub miner: Option<Arc<TokioRwLock<Miner<MainchainTransport>>>>,
     pub utxos: Arc<RwLock<HashMap<OutPoint, Output>>>,
     task: Arc<JoinHandle<()>>,
     pub transaction: Arc<RwLock<Transaction>>,
@@ -139,7 +152,7 @@ pub struct App {
 
 impl App {
     async fn task(
-        node: Arc<Node>,
+        node: Arc<Node<MainchainTransport>>,
         utxos: Arc<RwLock<HashMap<OutPoint, Output>>>,
         wallet: Wallet,
     ) -> Result<(), Error> {
@@ -151,7 +164,7 @@ impl App {
     }
 
     fn spawn_task(
-        node: Arc<Node>,
+        node: Arc<Node<MainchainTransport>>,
         utxos: Arc<RwLock<HashMap<OutPoint, Output>>>,
         wallet: Wallet,
     ) -> JoinHandle<()> {
@@ -162,7 +175,7 @@ impl App {
     }
 
     async fn check_status_serving(
-        client: &mut HealthClient<tonic::transport::Channel>,
+        client: &mut HealthClient<MainchainTransport>,
         service_name: &str,
     ) -> Result<bool, tonic::Status> {
         match client
@@ -194,9 +207,9 @@ impl App {
     /// Returns `Ok(_)` iff validator service is available, and error if validator
     /// service is unavailable.
     async fn check_proto_support(
-        transport: tonic::transport::channel::Channel,
+        transport: MainchainTransport,
     ) -> Result<ProtoSupport, tonic::Status> {
-        let mut client = HealthClient::new(transport);
+        let mut client = HealthClient::new(transport.clone());
 
         let block_producer_service_name =
             block_producer_service_server::SERVICE_NAME;
@@ -205,12 +218,34 @@ impl App {
         let wallet_service_name = wallet_service_server::SERVICE_NAME;
 
         // The validator service MUST exist. We therefore error out here directly.
-        if !Self::check_status_serving(&mut client, validator_service_name)
-            .await?
+        match Self::check_status_serving(&mut client, validator_service_name)
+            .await
         {
-            return Err(tonic::Status::aborted(format!(
-                "{validator_service_name} is not supported in mainchain client",
-            )));
+            Ok(true) => (),
+            Ok(false) => {
+                return Err(tonic::Status::aborted(format!(
+                    "{validator_service_name} is not supported in mainchain client",
+                )));
+            }
+            // Some proxies expose only the CUSF services. Call the validator
+            // directly instead; the optional services count as missing.
+            Err(health_err) => {
+                tracing::warn!(
+                    "Mainchain health service unavailable ({health_err}), calling {validator_service_name} directly"
+                );
+                let _tip = ValidatorServiceClient::new(transport)
+                    .get_chain_tip(GetChainTipRequest {})
+                    .await?;
+                tracing::info!(
+                    "Verified existence of {}",
+                    validator_service_name
+                );
+                return Ok(ProtoSupport {
+                    block_producer: false,
+                    miner: false,
+                    wallet: false,
+                });
+            }
         }
 
         tracing::info!("Verified existence of {}", validator_service_name);
@@ -264,12 +299,27 @@ impl App {
             config.mainchain_grpc_url
         );
         let rt_guard = runtime.enter();
-        let transport = tonic::transport::channel::Channel::from_shared(
+        let mut endpoint = tonic::transport::channel::Channel::from_shared(
             format!("{}", config.mainchain_grpc_url),
         )
         .unwrap()
-        .concurrency_limit(256)
-        .connect_lazy();
+        .concurrency_limit(256);
+        if config.mainchain_grpc_url.scheme() == "https" {
+            let mut tls =
+                tonic::transport::ClientTlsConfig::new().with_webpki_roots();
+            if let Some(path) = &config.mainchain_grpc_ca_cert {
+                tls = tls.ca_certificate(
+                    tonic::transport::Certificate::from_pem(std::fs::read(
+                        path,
+                    )?),
+                );
+            }
+            endpoint = endpoint.tls_config(tls).map_err(Error::MainchainTls)?;
+        }
+        let transport = crate::path_prefix::PathPrefix::new(
+            endpoint.connect_lazy(),
+            config.mainchain_grpc_url.path(),
+        );
         let (
             cusf_mainchain,
             cusf_mainchain_miner,

app/cli.rs

@@ -21,6 +21,7 @@ pub struct Config {
     pub log_level: tracing::Level,
     pub log_level_file: tracing::Level, // Level for logs that get written to file
     pub mainchain_grpc_url: url::Url,
+    pub mainchain_grpc_ca_cert: Option<PathBuf>,
     pub mnemonic_seed_phrase_path: Option<PathBuf>,
     pub net_addr: SocketAddr,
     pub network: Network,
@@ -43,6 +44,7 @@ impl Config {
             log_level,
             log_level_file,
             mainchain_grpc_url,
+            mainchain_grpc_ca_cert,
             mnemonic_seed_phrase_path,
             net_addr,
             network,
@@ -69,6 +71,8 @@ impl Config {
             %log_level,
             %log_level_file,
             %mainchain_grpc_url,
+            mainchain_grpc_ca_cert = mainchain_grpc_ca_cert.as_ref()
+                .map(|path| tracing::field::display(path.display())),
             mnemonic_seed_phrase_path = mnemonic_seed_phrase_path.as_ref()
                 .map(|path| tracing::field::display(path.display())),
             %net_addr,
@@ -206,10 +210,16 @@ pub(super) struct Cli {
     #[arg(default_value_t = tracing::Level::DEBUG, long)]
     log_level: tracing::Level,
 
-    /// Connect to mainchain node gRPC server running at this URL
+    /// Connect to mainchain node gRPC server running at this URL.
+    /// `https` URLs are verified against the Mozilla root certificates.
     #[arg(default_value = "http://localhost:50051", long)]
     mainchain_grpc_url: url::Url,
 
+    /// Also trust this CA certificate (PEM) for an `https` mainchain gRPC
+    /// URL, eg. for a mainchain node with a private CA
+    #[arg(long)]
+    mainchain_grpc_ca_cert: Option<PathBuf>,
+
     /// Path to a mnemonic seed phrase
     #[arg(long)]
     mnemonic_seed_phrase_path: Option<PathBuf>,
@@ -261,6 +271,13 @@ impl Cli {
             saturating_pred_level(self.log_level)
         };
         let wallet_dir = self.wallet_dir.unwrap_or_else(|| datadir.clone());
+        if self.mainchain_grpc_ca_cert.is_some()
+            && self.mainchain_grpc_url.scheme() != "https"
+        {
+            anyhow::bail!(
+                "--mainchain-grpc-ca-cert needs an https --mainchain-grpc-url"
+            );
+        }
         Ok(Config {
             add_peers: HashSet::from_iter(self.add_peers),
             datadir,
@@ -269,6 +286,7 @@ impl Cli {
             log_level,
             log_level_file: self.log_level_file,
             mainchain_grpc_url: self.mainchain_grpc_url,
+            mainchain_grpc_ca_cert: self.mainchain_grpc_ca_cert,
             mnemonic_seed_phrase_path: self.mnemonic_seed_phrase_path,
             net_addr: self.net_addr,
             network: self.network,

app/main.rs

@@ -11,6 +11,7 @@ mod app;
 mod cli;
 mod gui;
 mod line_buffer;
+mod path_prefix;
 mod rpc_server;
 mod util;
 
@@ -223,6 +224,7 @@ fn main() -> anyhow::Result<()> {
         add_peers: config.add_peers,
         datadir: config.datadir,
         mainchain_grpc_url: config.mainchain_grpc_url,
+        mainchain_grpc_ca_cert: config.mainchain_grpc_ca_cert,
         mnemonic_seed_phrase_path: config.mnemonic_seed_phrase_path,
         net_addr: config.net_addr,
         network: config.network,

app/path_prefix.rs

@@ -0,0 +1,57 @@
+//! gRPC transport that keeps the path of the mainchain URL
+
+use std::{
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+/// Prepends a path to every request, so the mainchain node can be served
+/// under one, eg. `https://example.com/enforcer`. tonic itself only keeps
+/// the URL's scheme and authority.
+#[derive(Clone, Debug)]
+pub struct PathPrefix<S> {
+    inner: S,
+    /// Without a trailing slash; `None` for the root
+    prefix: Option<Arc<str>>,
+}
+
+impl<S> PathPrefix<S> {
+    pub fn new(inner: S, prefix: &str) -> Self {
+        let prefix = prefix.trim_end_matches('/');
+        Self {
+            inner,
+            prefix: (!prefix.is_empty()).then(|| prefix.into()),
+        }
+    }
+}
+
+impl<S, B> tower::Service<http::Request<B>> for PathPrefix<S>
+where
+    S: tower::Service<http::Request<B>>,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = S::Future;
+
+    fn poll_ready(
+        &mut self,
+        cx: &mut Context<'_>,
+    ) -> Poll<Result<(), S::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: http::Request<B>) -> Self::Future {
+        if let Some(prefix) = &self.prefix {
+            let mut parts = request.uri().clone().into_parts();
+            let path_and_query =
+                parts.path_and_query.as_ref().map_or("/", |pq| pq.as_str());
+            parts.path_and_query =
+                Some(format!("{prefix}{path_and_query}").parse().expect(
+                    "a URL path followed by a gRPC path is a valid path",
+                ));
+            *request.uri_mut() =
+                http::Uri::from_parts(parts).expect("only the path changed");
+        }
+        self.inner.call(request)
+    }
+}

integration_tests/Cargo.toml

@@ -14,16 +14,21 @@ bitcoin = { workspace = true }
 clap = { workspace = true }
 dotenvy = { workspace = true }
 futures = { workspace = true }
+http-body-util = "0.1"
+hyper = { version = "1", features = ["client", "http2", "server"] }
+hyper-util = { version = "0.1", features = ["client-legacy", "http2", "tokio"] }
 jsonrpsee = { workspace = true }
 libtest-mimic = { workspace = true }
+rcgen = { workspace = true }
 reserve-port = { workspace = true }
 thiserror = { workspace = true }
 thunder = { path = "../lib", features = ["clap"] }
 thunder_app_rpc_api = { path = "../rpc-api" }
 tracing = { workspace = true }
 tracing-indicatif = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
-tokio = { workspace = true, features = ["rt-multi-thread"] }
+tokio = { workspace = true, features = ["net", "rt-multi-thread"] }
+tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] }
 
 [features]
 default = ["rustls"]

integration_tests/block_template.rs

@@ -52,10 +52,7 @@ async fn block_template_task(
     let mut enforcer_post_setup =
         setup(&bin_paths.others, res_tx.clone()).await?;
     let sidechain = PostSetup::setup(
-        Init {
-            thunder_app: bin_paths.thunder()?.clone(),
-            data_dir_suffix: None,
-        },
+        Init::new(bin_paths.thunder()?.clone()),
         &enforcer_post_setup,
         res_tx,
     )