feat: add neptune-core rpc connection watchdog

closes #1

Implements a background watchdog task that:
1. calls neptune-core /network rpc every N seconds (default: 10)
2. emits log message on any state change
3. sends detailed email alert to admin on any state change

Changes:
* AppState now has internal Arc<RwLock<AppStateInner>> to permit
  watchdog task to mutate the rpc_client field.
* adjust application to AppState changes (use locks)
* not_found functions no longer accept state arg
* move state initialization into AppState::init()
* add optional alert parameters: admin-email, smtp-*
* add neptune_rpc module with watchdog task
* add alert_email module
* simplify main()
* log warnings if alert parameters not set
* add chrono dep
* add lettre dep

Cargo.lock

@@ -376,6 +376,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "bech32"
 version = "0.9.1"
@@ -527,6 +533,16 @@ dependencies = [
  "windows-targets 0.52.5",
 ]
 
+[[package]]
+name = "chumsky"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9"
+dependencies = [
+ "hashbrown 0.14.3",
+ "stacker",
+]
+
 [[package]]
 name = "ciborium"
 version = "0.2.2"
@@ -707,6 +723,16 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
 
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.6"
@@ -977,6 +1003,22 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2"
 
+[[package]]
+name = "email-encoding"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60d1d33cdaede7e24091f039632eb5d3c7469fe5b066a985281a34fc70fa317f"
+dependencies = [
+ "base64 0.22.1",
+ "memchr",
+]
+
+[[package]]
+name = "email_address"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2153bd83ebc09db15bcbdc3e2194d901804952e3dc96967e1cd3b0c5c32d112"
+
 [[package]]
 name = "enum-ordinalize"
 version = "3.1.15"
@@ -1053,6 +1095,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -1259,7 +1316,7 @@ version = "7.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "byteorder",
  "flate2",
  "nom",
@@ -1299,6 +1356,17 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "hostname"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "windows",
+]
+
 [[package]]
 name = "html-escaper"
 version = "0.2.0"
@@ -1604,6 +1672,34 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "lettre"
+version = "0.11.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a62049a808f1c4e2356a2a380bd5f2aca3b011b0b482cf3b914ba1731426969"
+dependencies = [
+ "async-trait",
+ "base64 0.22.1",
+ "chumsky",
+ "email-encoding",
+ "email_address",
+ "fastrand",
+ "futures-io",
+ "futures-util",
+ "hostname",
+ "httpdate",
+ "idna",
+ "mime",
+ "native-tls",
+ "nom",
+ "percent-encoding",
+ "quoted_printable",
+ "socket2",
+ "tokio",
+ "tokio-native-tls",
+ "url",
+]
+
 [[package]]
 name = "leveldb-sys"
 version = "2.0.9"
@@ -1743,6 +1839,24 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "ndarray"
 version = "0.15.6"
@@ -1822,8 +1936,10 @@ dependencies = [
  "anyhow",
  "axum 0.7.5",
  "boilerplate",
+ "chrono",
  "clap",
  "html-escaper",
+ "lettre",
  "neptune-core",
  "readonly",
  "serde",
@@ -1987,6 +2103,50 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
+[[package]]
+name = "openssl"
+version = "0.10.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
+dependencies = [
+ "bitflags 2.5.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.58",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.102"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.18.0"
@@ -2160,6 +2320,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+
 [[package]]
 name = "plotters"
 version = "0.3.5"
@@ -2328,6 +2494,15 @@ dependencies = [
  "prost",
 ]
 
+[[package]]
+name = "psm"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -2366,6 +2541,12 @@ dependencies = [
  "syn 2.0.58",
 ]
 
+[[package]]
+name = "quoted_printable"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79ec282e887b434b68c18fe5c121d38e72a5cf35119b59e54ec5b992ea9c8eb0"
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -2601,12 +2782,44 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "schannel"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "security-framework"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0"
+dependencies = [
+ "bitflags 2.5.0",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "semver"
 version = "1.0.22"
@@ -2766,6 +2979,19 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "stacker"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "psm",
+ "winapi",
+]
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -3082,6 +3308,16 @@ dependencies = [
  "syn 2.0.58",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-serde"
 version = "0.8.0"
@@ -3133,7 +3369,7 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum 0.6.20",
- "base64",
+ "base64 0.21.7",
  "bytes",
  "h2",
  "http 0.2.12",
@@ -3461,6 +3697,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -3596,6 +3838,16 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.5",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.52.0"

Cargo.toml

@@ -26,6 +26,8 @@ html-escaper = "0.2.0"
 tower-http = { version = "0.5.2", features = ["fs"] }
 readonly = "0.2.12"
 url = "2.5.0"
+lettre = {version = "0.11.7", features = ["tokio1-native-tls"]}
+chrono = "0.4.34"
 
 # only should be used inside main.rs, for the binary.
 anyhow = "1.0.86"

src/alert_email.rs

@@ -0,0 +1,57 @@
+use crate::model::app_state::AppState;
+use crate::model::config::Config;
+use crate::model::config::SmtpMode;
+use clap::Parser;
+use lettre::transport::smtp::authentication::Credentials;
+use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor};
+use tracing::{info, warn};
+
+pub fn can_send_alerts() -> bool {
+    Config::parse().alert_config().is_some()
+}
+
+pub async fn send(
+    state: &AppState,
+    subject: &str,
+    body: String,
+) -> std::result::Result<bool, anyhow::Error> {
+    match state.read().await.config.alert_config() {
+        None => {
+            warn!("Alert emails disabled.  alert not sent.  consider confiuring smtp parameters.  subject: {subject}");
+            Ok(false)
+        }
+        Some(alert) => {
+            let email = Message::builder()
+                .from(alert.smtp_from_email.parse()?)
+                .to(alert.admin_email.parse()?)
+                .subject(subject)
+                .body(body)?;
+
+            // Create SMTP client credentials using username and password
+            let creds = Credentials::new(alert.smtp_user.to_string(), alert.smtp_pass.to_string());
+
+            // Open a secure connection to the SMTP server, possibly using STARTTLS
+            let relay = match alert.smtp_mode {
+                SmtpMode::Starttls => {
+                    AsyncSmtpTransport::<Tokio1Executor>::starttls_relay(&alert.smtp_host)?
+                }
+                SmtpMode::Smtps => AsyncSmtpTransport::<Tokio1Executor>::relay(&alert.smtp_host)?,
+            };
+            let mailer = relay.credentials(creds).build();
+
+            // Attempt to send the email via the SMTP transport
+            match mailer.send(email).await {
+                Ok(_) => info!(
+                    "alert email sent successfully.  to: {}, subject: {subject}, smtp_host: {}",
+                    alert.admin_email, alert.smtp_host
+                ),
+                Err(e) => warn!(
+                    "error send alert email. to: {}, subject: {subject}, smtp_host: {}.  error: {:?}",
+                    alert.admin_email, alert.smtp_host, e
+                ),
+            }
+
+            Ok(true)
+        }
+    }
+}

src/html/component/header.rs

@@ -1,9 +1,8 @@
-use crate::model::app_state::AppState;
+use crate::model::app_state::AppStateInner;
 use html_escaper::Escape;
-use std::sync::Arc;
 
 #[derive(boilerplate::Boilerplate)]
 #[boilerplate(filename = "web/html/components/header.html")]
-pub struct HeaderHtml {
+pub struct HeaderHtml<'a> {
-    pub state: Arc<AppState>,
+    pub state: &'a AppStateInner,
 }

src/html/page/block.rs

@@ -16,36 +16,34 @@ use tarpc::context;
 #[axum::debug_handler]
 pub async fn block_page(
     user_input_maybe: Result<Path<BlockSelectorExtended>, PathRejection>,
-    State(state): State<Arc<AppState>>,
+    State(state_rw): State<Arc<AppState>>,
 ) -> Result<Html<String>, Response> {
     #[derive(boilerplate::Boilerplate)]
     #[boilerplate(filename = "web/html/page/block_info.html")]
-    pub struct BlockInfoHtmlPage {
+    pub struct BlockInfoHtmlPage<'a> {
-        header: HeaderHtml,
+        header: HeaderHtml<'a>,
         block_info: BlockInfo,
     }
+    let state = &*state_rw.read().await;
 
-    let Path(block_selector) = user_input_maybe
+    let Path(block_selector) =
-        .map_err(|e| not_found_html_response(State(state.clone()), Some(e.to_string())))?;
+        user_input_maybe.map_err(|e| not_found_html_response(state, Some(e.to_string())))?;
-
-    let header = HeaderHtml {
-        state: state.clone(),
-    };
 
     let block_info = match state
-        .clone()
         .rpc_client
         .block_info(context::current(), block_selector.into())
         .await
-        .map_err(|e| not_found_html_response(State(state.clone()), Some(e.to_string())))?
+        .map_err(|e| not_found_html_response(state, Some(e.to_string())))?
     {
         Some(info) => Ok(info),
         None => Err(not_found_html_response(
-            State(state),
+            state,
             Some("Block does not exist".to_string()),
         )),
     }?;
 
+    let header = HeaderHtml { state };
+
     let block_info_page = BlockInfoHtmlPage { header, block_info };
     Ok(Html(block_info_page.to_string()))
 }

src/html/page/not_found.rs

@@ -1,46 +1,30 @@
-use crate::html::component::header::HeaderHtml;
 use crate::http_util::not_found_html_err;
 use crate::http_util::not_found_html_handler;
-use crate::model::app_state::AppState;
+use crate::model::app_state::AppStateInner;
-use axum::extract::State;
 use axum::http::StatusCode;
 use axum::response::Html;
 use axum::response::Response;
 use html_escaper::Escape;
-// use html_escaper::Trusted;
-use std::sync::Arc;
 
-// #[axum::debug_handler]
+pub fn not_found_page(error_msg: Option<String>) -> Html<String> {
-pub fn not_found_page(
-    State(state): State<Arc<AppState>>,
-    error_msg: Option<String>,
-) -> Html<String> {
     #[derive(boilerplate::Boilerplate)]
     #[boilerplate(filename = "web/html/page/not_found.html")]
     #[allow(dead_code)]
     pub struct NotFoundHtmlPage {
-        header: HeaderHtml,
         error_msg: String,
     }
 
-    let header = HeaderHtml {
-        state: state.clone(),
-    };
-
     let not_found_page = NotFoundHtmlPage {
-        header,
         error_msg: error_msg.unwrap_or_default(),
     };
     Html(not_found_page.to_string())
 }
 
-pub fn not_found_html_response(
+pub fn not_found_html_response(_state: &AppStateInner, error_msg: Option<String>) -> Response {
-    State(state): State<Arc<AppState>>,
+    not_found_html_err(not_found_page(error_msg))
-    error_msg: Option<String>,
-) -> Response {
-    not_found_html_err(not_found_page(State(state), error_msg))
 }
 
-pub fn not_found_html_fallback(state: Arc<AppState>) -> (StatusCode, Html<String>) {
+#[axum::debug_handler]
-    not_found_html_handler(not_found_page(State(state), None))
+pub async fn not_found_html_fallback() -> (StatusCode, Html<String>) {
+    not_found_html_handler(not_found_page(None))
 }

src/html/page/redirect_qs_to_path.rs

@@ -74,9 +74,11 @@ use crate::model::app_state::AppState;
 #[axum::debug_handler]
 pub async fn redirect_query_string_to_path(
     RawQuery(raw_query_option): RawQuery,
-    State(state): State<Arc<AppState>>,
+    State(state_rw): State<Arc<AppState>>,
 ) -> Result<Response, Response> {
-    let not_found = || not_found_html_response(State(state.clone()), None);
+    let state = &*state_rw.read().await;
+
+    let not_found = || not_found_html_response(state, None);
 
     let raw_query = raw_query_option.ok_or_else(not_found)?;
 

src/html/page/root.rs

@@ -1,5 +1,6 @@
 use crate::html::page::not_found::not_found_html_response;
 use crate::model::app_state::AppState;
+use crate::model::app_state::AppStateInner;
 use axum::extract::State;
 use axum::response::Html;
 use axum::response::Response;
@@ -9,19 +10,21 @@ use std::sync::Arc;
 use tarpc::context;
 
 #[axum::debug_handler]
-pub async fn root(State(state): State<Arc<AppState>>) -> Result<Html<String>, Response> {
+pub async fn root(State(state_rw): State<Arc<AppState>>) -> Result<Html<String>, Response> {
     #[derive(boilerplate::Boilerplate)]
     #[boilerplate(filename = "web/html/page/root.html")]
-    pub struct RootHtmlPage {
+    pub struct RootHtmlPage<'a> {
         tip_height: BlockHeight,
-        state: Arc<AppState>,
+        state: &'a AppStateInner,
     }
 
+    let state = &*state_rw.read().await;
+
     let tip_height = state
         .rpc_client
         .block_height(context::current())
         .await
-        .map_err(|e| not_found_html_response(State(state.clone()), Some(e.to_string())))?;
+        .map_err(|e| not_found_html_response(state, Some(e.to_string())))?;
 
     let root_page = RootHtmlPage { tip_height, state };
     Ok(Html(root_page.to_string()))

src/html/page/utxo.rs

@@ -15,37 +15,37 @@ use tarpc::context;
 #[axum::debug_handler]
 pub async fn utxo_page(
     index_maybe: Result<Path<u64>, PathRejection>,
-    State(state): State<Arc<AppState>>,
+    State(state_rw): State<Arc<AppState>>,
 ) -> Result<Html<String>, Response> {
     #[derive(boilerplate::Boilerplate)]
     #[boilerplate(filename = "web/html/page/utxo.html")]
-    pub struct UtxoHtmlPage {
+    pub struct UtxoHtmlPage<'a> {
-        header: HeaderHtml,
+        header: HeaderHtml<'a>,
         index: u64,
         digest: Digest,
     }
 
-    let Path(index) = index_maybe
+    let state = &*state_rw.read().await;
-        .map_err(|e| not_found_html_response(State(state.clone()), Some(e.to_string())))?;
+
+    let Path(index) =
+        index_maybe.map_err(|e| not_found_html_response(state, Some(e.to_string())))?;
 
     let digest = match state
         .rpc_client
         .utxo_digest(context::current(), index)
         .await
-        .map_err(|e| not_found_html_response(State(state.clone()), Some(e.to_string())))?
+        .map_err(|e| not_found_html_response(state, Some(e.to_string())))?
     {
         Some(digest) => digest,
         None => {
             return Err(not_found_html_response(
-                State(state.clone()),
+                state,
                 Some("The requested UTXO does not exist".to_string()),
             ))
         }
     };
 
-    let header = HeaderHtml {
+    let header = HeaderHtml { state };
-        state: state.clone(),
-    };
 
     let utxo_page = UtxoHtmlPage {
         index,

src/http_util.rs

@@ -20,5 +20,5 @@ pub fn not_found_html_handler(html: Html<String>) -> (StatusCode, Html<String>)
 }
 
 pub fn rpc_err(e: RpcError) -> Response {
-    (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response()
+    (StatusCode::INTERNAL_SERVER_ERROR, format!("{:?}", e)).into_response()
 }

src/lib.rs

@@ -1,4 +1,6 @@
+pub mod alert_email;
 pub mod html;
 pub mod http_util;
 pub mod model;
+pub mod neptune_rpc;
 pub mod rpc;

src/main.rs

@@ -1,50 +1,54 @@
 use anyhow::Context;
 use axum::routing::get;
 use axum::routing::Router;
-use clap::Parser;
+use neptune_explorer::alert_email;
-use neptune_core::models::blockchain::block::block_selector::BlockSelector;
-use neptune_core::rpc_server::RPCClient;
 use neptune_explorer::html::page::block::block_page;
 use neptune_explorer::html::page::not_found::not_found_html_fallback;
 use neptune_explorer::html::page::redirect_qs_to_path::redirect_query_string_to_path;
 use neptune_explorer::html::page::root::root;
 use neptune_explorer::html::page::utxo::utxo_page;
 use neptune_explorer::model::app_state::AppState;
-use neptune_explorer::model::config::Config;
+use neptune_explorer::neptune_rpc;
 use neptune_explorer::rpc::block_digest::block_digest;
 use neptune_explorer::rpc::block_info::block_info;
 use neptune_explorer::rpc::utxo_digest::utxo_digest;
-use std::net::Ipv4Addr;
-use std::net::SocketAddr;
-use std::sync::Arc;
-use tarpc::client;
-use tarpc::context;
-use tarpc::tokio_serde::formats::Json as RpcJson;
 use tower_http::services::ServeDir;
+use tracing::{info, warn};
+use tracing_subscriber::EnvFilter;
 
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
-    let rpc_client = rpc_client()
+    let filter = EnvFilter::from_default_env()
-        .await
+        // Set the base level when not matched by other directives to INFO.
-        .with_context(|| "Failed to create RPC client")?;
+        .add_directive("neptune_explorer=info".parse()?);
-    let network = rpc_client
-        .network(context::current())
-        .await
-        .with_context(|| "Failed calling neptune-core api: network")?;
-    let genesis_digest = rpc_client
-        .block_digest(context::current(), BlockSelector::Genesis)
-        .await
-        .with_context(|| "Failed calling neptune-core api: block_digest")?
-        .with_context(|| "neptune-core failed to provide a genesis block")?;
 
-    let shared_state = Arc::new(AppState::from((
+    tracing_subscriber::fmt().with_env_filter(filter).init();
-        network,
-        Config::parse(),
-        rpc_client,
-        genesis_digest,
-    )));
 
-    let app = Router::new()
+    let app_state = AppState::init().await?;
+
+    let routes = setup_routes(app_state.clone());
+
+    let port = app_state.read().await.config.listen_port;
+    let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))
+        .await
+        .with_context(|| format!("Failed to bind to port {port}"))?;
+
+    if !alert_email::can_send_alerts() {
+        warn!("alert emails disabled.  consider configuring smtp parameters.");
+    }
+
+    tokio::task::spawn(neptune_rpc::watchdog(app_state));
+
+    info!("Running on http://localhost:{port}");
+
+    axum::serve(listener, routes)
+        .await
+        .with_context(|| "Axum server encountered an error")?;
+    Ok(())
+}
+
+pub fn setup_routes(app_state: AppState) -> Router {
+    Router::new()
         // -- RPC calls --
         .route("/rpc/block_info/*selector", get(block_info))
         .route("/rpc/block_digest/*selector", get(block_digest))
@@ -65,34 +69,7 @@ async fn main() -> Result<(), anyhow::Error> {
             ServeDir::new(concat!(env!("CARGO_MANIFEST_DIR"), "/templates/web/image")),
         )
         // handle route not-found
-        .fallback(not_found_html_fallback(shared_state.clone()))
+        .fallback(not_found_html_fallback)
         // add state
-        .with_state(shared_state);
+        .with_state(app_state.into())
-
-    let port = Config::parse().listen_port;
-    let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))
-        .await
-        .with_context(|| format!("Failed to bind to port {port}"))?;
-
-    println!("Running on http://localhost:{port}");
-
-    axum::serve(listener, app)
-        .await
-        .with_context(|| "Axum server encountered an error")?;
-    Ok(())
-}
-
-async fn rpc_client() -> Result<RPCClient, anyhow::Error> {
-    // Create connection to neptune-core RPC server
-    let args: Config = Config::parse();
-    let server_socket = SocketAddr::new(
-        std::net::IpAddr::V4(Ipv4Addr::LOCALHOST),
-        args.neptune_rpc_port,
-    );
-    let transport = tarpc::serde_transport::tcp::connect(server_socket, RpcJson::default)
-        .await
-        .with_context(|| {
-            format!("Failed to connect to neptune-core rpc service at {server_socket}")
-        })?;
-    Ok(RPCClient::new(client::Config::default(), transport).spawn())
 }

src/model/app_state.rs

@@ -1,25 +1,65 @@
 use crate::model::config::Config;
+use crate::neptune_rpc;
+use anyhow::Context;
+use clap::Parser;
 use neptune_core::config_models::network::Network;
+use neptune_core::models::blockchain::block::block_selector::BlockSelector;
 use neptune_core::prelude::twenty_first::math::digest::Digest;
 use neptune_core::rpc_server::RPCClient;
+use std::sync::Arc;
+use tokio::sync::RwLock;
 
-#[readonly::make]
+pub struct AppStateInner {
-pub struct AppState {
     pub network: Network,
     pub config: Config,
     pub rpc_client: RPCClient,
     pub genesis_digest: Digest,
 }
 
+#[derive(Clone)]
+pub struct AppState(Arc<RwLock<AppStateInner>>);
+
+impl std::ops::Deref for AppState {
+    type Target = Arc<RwLock<AppStateInner>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
 impl From<(Network, Config, RPCClient, Digest)> for AppState {
     fn from(
         (network, config, rpc_client, genesis_digest): (Network, Config, RPCClient, Digest),
     ) -> Self {
-        Self {
+        Self(Arc::new(RwLock::new(AppStateInner {
             network,
             config,
             rpc_client,
             genesis_digest,
+        })))
     }
 }
+
+impl AppState {
+    pub async fn init() -> Result<Self, anyhow::Error> {
+        let rpc_client = neptune_rpc::gen_rpc_client()
+            .await
+            .with_context(|| "Failed to create RPC client")?;
+        let network = rpc_client
+            .network(tarpc::context::current())
+            .await
+            .with_context(|| "Failed calling neptune-core api: network")?;
+        let genesis_digest = rpc_client
+            .block_digest(tarpc::context::current(), BlockSelector::Genesis)
+            .await
+            .with_context(|| "Failed calling neptune-core api: block_digest")?
+            .with_context(|| "neptune-core failed to provide a genesis block")?;
+
+        Ok(Self::from((
+            network,
+            Config::parse(),
+            rpc_client,
+            genesis_digest,
+        )))
+    }
 }

src/model/config.rs

@@ -1,16 +1,124 @@
+// Config and AlertConfig seem more complex than they should be.
+// The requirement is that alert params are all-or-none, but
+// Clap makes it non-obvious / difficult to express that.
+//
+// There is a discussion about it:
+// https://github.com/clap-rs/clap/discussions/5506
+//
+// Hopefully we can simplify it soon / eventually.
+
 #[readonly::make]
 #[derive(Debug, clap::Parser, Clone)]
 #[clap(name = "neptune-explorer", about = "Neptune Block Explorer")]
+#[clap(group(
+    clap::ArgGroup::new("value")
+        .required(false)
+        .multiple(true)
+        .requires_all(&["admin_email", "smtp_host", "smtp_user", "smtp_pass", "smtp_from_email"])
+        .args(&["admin_email", "smtp_host", "smtp_user", "smtp_pass", "smtp_from_email"])
+))]
 pub struct Config {
     /// Sets the website name
     #[clap(long, default_value = "Neptune Explorer", value_name = "site-name")]
     pub site_name: String,
 
+    /// Sets the website domain, eg 'explorer.mydomain.com'. used for alert emails, etc.
+    #[clap(long, value_name = "domain")]
+    pub site_domain: String,
+
     /// Sets the port to listen for http requests.
-    #[clap(long, default_value = "3000", value_name = "PORT")]
+    #[clap(long, default_value = "3000", value_name = "port")]
     pub listen_port: u16,
 
     /// Sets the neptune-core rpc server address to connect to.
-    #[clap(long, default_value = "9799", value_name = "PORT")]
+    #[clap(long, default_value = "9799", value_name = "port")]
     pub neptune_rpc_port: u16,
+
+    /// Sets interval in seconds to ping neptune-core rpc connection
+    #[clap(long, default_value = "10", value_name = "seconds")]
+    pub neptune_rpc_watchdog_secs: u64,
+
+    /// admin email for receiving alert emails
+    #[arg(long, value_name = "email")]
+    pub admin_email: Option<String>,
+
+    /// smtp host for alert emails
+    #[arg(long, value_name = "host")]
+    pub smtp_host: Option<String>,
+
+    /// smtp username for alert emails
+    #[arg(long, value_name = "user")]
+    pub smtp_user: Option<String>,
+
+    /// smtp password for alert emails
+    #[arg(long, value_name = "pass")]
+    pub smtp_pass: Option<String>,
+
+    /// sender email for alerts
+    #[arg(long, value_name = "email")]
+    pub smtp_from_email: Option<String>,
+
+    /// connect with smtps or starttls
+    #[arg(long, value_enum, value_name = "mode", default_value = "smtps")]
+    pub smtp_mode: SmtpMode,
+}
+
+impl Config {
+    pub fn alert_config(&self) -> Option<AlertConfig> {
+        match (
+            &self.admin_email,
+            &self.smtp_host,
+            &self.smtp_user,
+            &self.smtp_pass,
+            &self.smtp_from_email,
+            &self.smtp_mode,
+        ) {
+            (
+                Some(admin_email),
+                Some(smtp_host),
+                Some(smtp_user),
+                Some(smtp_pass),
+                Some(smtp_from_email),
+                smtp_mode,
+            ) => Some(AlertConfig {
+                admin_email: admin_email.clone(),
+                smtp_host: smtp_host.clone(),
+                smtp_user: smtp_user.clone(),
+                smtp_pass: smtp_pass.clone(),
+                smtp_from_email: smtp_from_email.clone(),
+                smtp_mode: smtp_mode.clone(),
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, clap::Args)]
+pub struct AlertConfig {
+    /// admin email for receiving alert emails
+    pub admin_email: String,
+
+    /// smtp host for alert emails
+    pub smtp_host: String,
+
+    /// smtp username for alert emails
+    pub smtp_user: String,
+
+    /// smtp password for alert emails
+    pub smtp_pass: String,
+
+    /// sender email for alerts
+    pub smtp_from_email: String,
+
+    /// connect with smtps or starttls
+    pub smtp_mode: SmtpMode,
+}
+
+#[derive(Debug, Clone, clap::ValueEnum)]
+pub enum SmtpMode {
+    /// smtps
+    Smtps,
+
+    /// starttls
+    Starttls,
 }

src/neptune_rpc.rs

@@ -0,0 +1,115 @@
+use crate::alert_email;
+use crate::model::app_state::AppState;
+use crate::model::config::Config;
+use anyhow::Context;
+use chrono::DateTime;
+use chrono::TimeDelta;
+use chrono::Utc;
+use clap::Parser;
+use neptune_core::rpc_server::RPCClient;
+use std::net::Ipv4Addr;
+use std::net::SocketAddr;
+use tarpc::client;
+use tarpc::context;
+use tarpc::tokio_serde::formats::Json as RpcJson;
+use tracing::{debug, info, warn};
+
+/// generates RPCClient, for querying neptune-core RPC server.
+pub async fn gen_rpc_client() -> Result<RPCClient, anyhow::Error> {
+    // Create connection to neptune-core RPC server
+    let args: Config = Config::parse();
+    let server_socket = SocketAddr::new(
+        std::net::IpAddr::V4(Ipv4Addr::LOCALHOST),
+        args.neptune_rpc_port,
+    );
+    let transport = tarpc::serde_transport::tcp::connect(server_socket, RpcJson::default)
+        .await
+        .with_context(|| {
+            format!("Failed to connect to neptune-core rpc service at {server_socket}")
+        })?;
+    Ok(RPCClient::new(client::Config::default(), transport).spawn())
+}
+
+/// a tokio task that periodically pings neptune-core rpc server to ensure the
+/// connection is still alive and/or attempts to re-establish connection.
+///
+/// If not connected, a single connection attempt is made for each timer iteration.
+///
+/// Whenever the connection changes state a log message is printed and an email
+/// alert is sent to admin, if admin_email config field is set.  In this way,
+/// the site admin gets notified of both outages and restoration of service.
+pub async fn watchdog(app_state: AppState) {
+    let app_started = chrono::offset::Utc::now();
+    let mut was_connected = true;
+    let mut since = chrono::offset::Utc::now();
+    let watchdog_secs = app_state.read().await.config.neptune_rpc_watchdog_secs;
+
+    debug!("neptune-core rpc watchdog started");
+
+    loop {
+        tokio::time::sleep(tokio::time::Duration::from_secs(watchdog_secs)).await;
+
+        let result = app_state
+            .read()
+            .await
+            .rpc_client
+            .network(context::current())
+            .await;
+
+        let now_connected = result.is_ok();
+        if now_connected != was_connected {
+            // send admin alert of state change.
+            let subject = match now_connected {
+                true => "alert!  ** RECOVERY ** rpc connection restored",
+                false => "alert!  ** OUTAGE ** rpc connection lost.",
+            };
+
+            let config = Config::parse();
+            let now = chrono::offset::Utc::now();
+            let duration = now.signed_duration_since(since);
+            let app_duration = now.signed_duration_since(app_started);
+            let body = NeptuneRpcAlertEmail {
+                config,
+                was_connected,
+                now_connected,
+                now,
+                app_started,
+                app_duration,
+                since,
+                duration,
+            }
+            .to_string();
+
+            let msg = format!("alert: neptune-core rpc connection status change: previous: {was_connected}, now: {now_connected}");
+            match now_connected {
+                true => info!("{msg}"),
+                false => warn!("{msg}"),
+            }
+
+            let _ = alert_email::send(&app_state, subject, body).await;
+
+            was_connected = now_connected;
+            since = chrono::offset::Utc::now();
+        }
+
+        if !now_connected {
+            if let Ok(c) = gen_rpc_client().await {
+                let mut state = app_state.write().await;
+                state.rpc_client = c;
+            }
+        }
+    }
+}
+
+#[derive(boilerplate::Boilerplate)]
+#[boilerplate(filename = "email/neptune_rpc_alert.txt")]
+pub struct NeptuneRpcAlertEmail {
+    config: Config,
+    was_connected: bool,
+    now_connected: bool,
+    app_started: DateTime<Utc>,
+    app_duration: TimeDelta,
+    since: DateTime<Utc>,
+    now: DateTime<Utc>,
+    duration: TimeDelta,
+}

src/rpc/block_digest.rs

@@ -16,6 +16,8 @@ pub async fn block_digest(
     State(state): State<Arc<AppState>>,
 ) -> Result<Json<Digest>, impl IntoResponse> {
     match state
+        .read()
+        .await
         .rpc_client
         .block_digest(context::current(), selector.into())
         .await

src/rpc/block_info.rs

@@ -16,6 +16,8 @@ pub async fn block_info(
     State(state): State<Arc<AppState>>,
 ) -> Result<Json<BlockInfo>, Response> {
     let block_info = state
+        .read()
+        .await
         .rpc_client
         .block_info(context::current(), selector.into())
         .await

src/rpc/utxo_digest.rs

@@ -17,6 +17,8 @@ pub async fn utxo_digest(
     State(state): State<Arc<AppState>>,
 ) -> Result<Json<Digest>, impl IntoResponse> {
     match state
+        .read()
+        .await
         .rpc_client
         .utxo_digest(context::current(), index)
         .await

templates/email/neptune_rpc_alert.txt

@@ -0,0 +1,53 @@
+%% if self.now_connected {
+**** ALERT: Neptune RPC Connection Restored ****
+%% } else {
+**** ALERT: Neptune RPC Connection Lost ****
+%% }
+
+site: {{self.config.site_name}} at {{self.config.site_domain}}:{{self.config.listen_port}}
+
+-- Details --
+
+Event: Neptune RPC Connection Status Change.
+
+Event Time: {{self.now.to_rfc3339()}}
+
+Event Description:
+
+%% if self.now_connected {
+The neptune-explorer application has re-established connection with the
+neptune-core rpc server.  Service is restored.
+%% } else {
+The neptune-explorer application is unable to connect to the neptune-core rpc
+server.  Website users are experiencing a site-outage.
+%% }
+
+New Status:
+    Neptune RPC Connected: {{self.now_connected}}
+    Now: {{self.now.to_rfc3339()}}
+
+Previous Status:
+    Neptune RPC Connected: {{self.was_connected}}
+    Since: {{self.since.to_rfc3339()}}
+    Duration: {{self.duration}}
+
+Block Explorer Uptime:
+    Started: {{self.app_started.to_rfc3339()}}
+    Duration: {{self.app_duration}} seconds
+
+Neptune-core RPC:
+    Host: {{self.config.site_domain}}   (localhost)
+    Port: {{self.config.neptune_rpc_port}}
+
+Recommended action:
+
+%% if self.now_connected {
+    Test the block explorer to ensure it is operating correctly.
+
+    No further corrective action should be necessary.
+%% } else {
+    Review if neptune-core is running and accessible.  Restart if necessary.
+
+    When neptune-core becomes available the block-explorer should automatically
+    re-establish the connection within {{self.config.neptune_rpc_watchdog_secs}} seconds.
+%% }

templates/web/html/page/not_found.html

@@ -1,7 +1,7 @@
 <html>
 
 <head>
-    <title>{{self.header.state.config.site_name}}: Not Found</title>
+    <title>Not Found</title>
 {{html_escaper::Trusted(include_str!( concat!(env!("CARGO_MANIFEST_DIR"), "/templates/web/html/components/head.html")))}}
 </head>