diff --git a/Cargo.lock b/Cargo.lock index 9f7ba70..36b0e29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -526,6 +526,25 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "h2" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.11", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.1" @@ -537,7 +556,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 1.0.0", "indexmap", "slab", "tokio", @@ -569,7 +588,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 1.0.0", "httpdate", "mime", "sha1", @@ -581,7 +600,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.0.0", ] [[package]] @@ -608,6 +627,17 @@ dependencies = [ "digest", ] +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.0.0" @@ -619,6 +649,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.11", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.0" @@ -626,7 +667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http", + "http 1.0.0", ] [[package]] @@ -637,8 +678,8 @@ checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -654,6 +695,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.23", + "http 0.2.11", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.1.0" @@ -663,9 +728,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.1", + "http 1.0.0", + "http-body 1.0.0", "httparse", "httpdate", "itoa", @@ -682,7 +747,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.1.0", "hyper-util", "native-tls", "tokio", @@ -699,9 +764,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", "pin-project-lite", "socket2", "tokio", @@ -897,7 +962,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.0.0", "httparse", "log", "memchr", @@ -1187,10 +1252,10 @@ dependencies = [ "cookie", "futures-util", "headers", - "http", + "http 1.0.0", "http-body-util", "httpdate", - "hyper", + "hyper 1.1.0", "hyper-util", "mime", "mime_guess", @@ -1202,6 +1267,7 @@ dependencies = [ "poem-derive", "quick-xml", "regex", + "reqwest 0.11.23 (registry+https://github.com/rust-lang/crates.io-index)", "rfc7239", "serde", "serde_json", @@ -1265,7 +1331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5d485fb9cc4ca9a8364beedd4ea81294b1f028d459c8fd7bb352e38f87f8ffa" dependencies = [ "darling", - "http", + "http 1.0.0", "indexmap", "mime", "proc-macro-crate", @@ -1418,6 +1484,41 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.23", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "reqwest" version = "0.11.23" @@ -1428,11 +1529,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.1", + "http 1.0.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.1.0", "hyper-tls", "hyper-util", "ipnet", @@ -1472,7 +1573,7 @@ version = "0.1.0" dependencies = [ "config", "futures-util", - "http", + "http 1.0.0", "hyper-util", "lazy_static", "mime", @@ -1482,7 +1583,7 @@ dependencies = [ "queryst", "rand", "regex", - "reqwest", + "reqwest 0.11.23 (git+https://github.com/seanmonstar/reqwest.git?branch=hyper-v1)", "serde", "serde_json", "tokio", @@ -2123,7 +2224,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 1.0.0", "httparse", "log", "rand", diff --git a/Cargo.toml b/Cargo.toml index 9c7a19c..95fae7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ hyper-util = { version = "0.1.2", features = ["full"] } lazy_static = "1.4.0" mime = "0.3.17" percent-encoding = "2.3.1" -poem = { version = "2.0.0", features = ["tokio-metrics", "websocket", "static-files"] } +poem = { version = "2.0.0", features = ["tokio-metrics", "websocket", "static-files", "reqwest"] } poem-openapi = { version = "4.0.0", features = ["swagger-ui"] } queryst = "3.0.0" rand = "0.8.5" diff --git a/src/config/mod.rs b/src/config/mod.rs index c7507d4..de6edc1 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,7 +1,6 @@ -use std::sync::RwLock; - use config::Config; use lazy_static::lazy_static; +use tokio::sync::RwLock; use crate::config::loader::load_settings; diff --git a/src/main.rs b/src/main.rs index 2906943..27195b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,11 +2,28 @@ mod config; mod proxies; mod sideload; -use poem::{listener::TcpListener, EndpointExt, Route, Server}; +use std::collections::VecDeque; + +use lazy_static::lazy_static; +use poem::{listener::TcpListener, Route, Server}; use poem_openapi::OpenApiService; +use proxies::RoadInstance; +use tokio::sync::Mutex; use tracing::{error, info, Level}; -use crate::proxies::route; +use crate::proxies::{metrics::RoadMetrics, route}; + +lazy_static! { + static ref ROAD: Mutex = Mutex::new(RoadInstance { + regions: vec![], + metrics: RoadMetrics { + requests_count: 0, + failures_count: 0, + recent_successes: VecDeque::new(), + recent_errors: VecDeque::new(), + } + }); +} #[tokio::main] async fn main() -> Result<(), std::io::Error> { @@ -19,19 +36,17 @@ async fn main() -> Result<(), std::io::Error> { .init(); // Prepare all the stuff - let mut instance = proxies::Instance::new(); - info!("Loading proxy regions..."); match proxies::loader::scan_regions( config::C .read() - .unwrap() + .await .get_string("regions") .unwrap_or("./regions".to_string()), ) { Err(_) => error!("Loading proxy regions... failed"), Ok((regions, count)) => { - instance.regions = regions; + ROAD.lock().await.regions = regions; info!(count, "Loading proxy regions... done") } }; @@ -40,11 +55,11 @@ async fn main() -> Result<(), std::io::Error> { let proxies_server = Server::new(TcpListener::bind( config::C .read() - .unwrap() + .await .get_string("listen.proxies") .unwrap_or("0.0.0.0:80".to_string()), )) - .run(route::handle.data(instance)); + .run(route::handle); // Sideload let sideload = OpenApiService::new(sideload::SideloadApi, "Sideload API", "1.0") @@ -54,7 +69,7 @@ async fn main() -> Result<(), std::io::Error> { let sideload_server = Server::new(TcpListener::bind( config::C .read() - .unwrap() + .await .get_string("listen.sideload") .unwrap_or("0.0.0.0:81".to_string()), )) diff --git a/src/proxies/metrics.rs b/src/proxies/metrics.rs new file mode 100644 index 0000000..dad7275 --- /dev/null +++ b/src/proxies/metrics.rs @@ -0,0 +1,80 @@ +use std::collections::VecDeque; + +use poem_openapi::Object; +use serde::{Deserialize, Serialize}; + +use super::config::{Destination, Location, Region}; + +#[derive(Debug, Object, Clone, Serialize, Deserialize, PartialEq)] +pub struct RoadTrace { + pub region: String, + pub location: String, + pub destination: String, + pub error: Option, +} + +impl RoadTrace { + pub fn from_structs(reg: Region, loc: Location, end: Destination) -> RoadTrace { + RoadTrace { + region: reg.id, + location: loc.id, + destination: end.id, + error: None, + } + } + + pub fn from_structs_with_error( + reg: Region, + loc: Location, + end: Destination, + err: String, + ) -> RoadTrace { + RoadTrace { + region: reg.id, + location: loc.id, + destination: end.id, + error: Some(err), + } + } +} + +#[derive(Debug, Clone)] +pub struct RoadMetrics { + pub requests_count: u64, + pub failures_count: u64, + + pub recent_successes: VecDeque, + pub recent_errors: VecDeque, +} + +impl RoadMetrics { + pub fn get_success_rate(&self) -> f64 { + if self.requests_count > 0 { + (self.requests_count - self.failures_count) as f64 / self.requests_count as f64 + } else { + 0.0 + } + } + + pub fn add_success_request(&mut self, reg: Region, loc: Location, end: Destination) { + self.requests_count += 1; + self.recent_successes + .push_back(RoadTrace::from_structs(reg, loc, end)); + } + + pub fn add_faliure_request( + &mut self, + reg: Region, + loc: Location, + end: Destination, + err: String, // For some reason error is rarely clonable, so we use preformatted message + ) { + self.requests_count += 1; + self.failures_count += 1; + self.recent_errors + .push_back(RoadTrace::from_structs_with_error(reg, loc, end, err)); + if self.recent_errors.len() > 10 { + self.recent_errors.pop_front(); + } + } +} diff --git a/src/proxies/mod.rs b/src/proxies/mod.rs index aeb2178..2433bb8 100644 --- a/src/proxies/mod.rs +++ b/src/proxies/mod.rs @@ -1,30 +1,35 @@ use http::Method; use poem::http::{HeaderMap, Uri}; use regex::Regex; -use serde::{Deserialize, Serialize}; use wildmatch::WildMatch; -use self::config::{Location, Region}; +use self::{ + config::{Location, Region}, + metrics::RoadMetrics, +}; pub mod browser; pub mod config; pub mod loader; +pub mod metrics; pub mod responder; pub mod route; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Instance { +#[derive(Debug, Clone)] +pub struct RoadInstance { pub regions: Vec, + pub metrics: RoadMetrics, } -impl Instance { - pub fn new() -> Instance { - Instance { regions: vec![] } - } - - pub fn filter(&self, uri: &Uri, method: Method, headers: &HeaderMap) -> Option<&Location> { +impl RoadInstance { + pub fn filter( + &self, + uri: &Uri, + method: Method, + headers: &HeaderMap, + ) -> Option<(&Region, &Location)> { self.regions.iter().find_map(|region| { - region.locations.iter().find(|location| { + let location = region.locations.iter().find(|location| { let mut hosts = location.hosts.iter(); if !hosts.any(|item| { WildMatch::new(item.as_str()).matches(uri.host().unwrap_or("localhost")) @@ -64,7 +69,9 @@ impl Instance { } true - }) + }); + + location.map(|location| (region, location)) }) } } diff --git a/src/proxies/route.rs b/src/proxies/route.rs index 69c0be6..c45b43c 100644 --- a/src/proxies/route.rs +++ b/src/proxies/route.rs @@ -1,27 +1,30 @@ +use http::Method; use poem::{ handler, http::{HeaderMap, StatusCode, Uri}, - web::{websocket::WebSocket, Data}, + web::websocket::WebSocket, Body, Error, FromRequest, IntoResponse, Request, Response, Result, }; use rand::seq::SliceRandom; -use reqwest::Method; -use crate::proxies::{ - config::{Destination, DestinationType}, - responder, +use crate::{ + proxies::{ + config::{Destination, DestinationType}, + responder, + }, + ROAD, }; #[handler] pub async fn handle( - app: Data<&super::Instance>, req: &Request, uri: &Uri, headers: &HeaderMap, method: Method, body: Body, ) -> Result { - let location = match app.filter(uri, method.clone(), headers) { + let readable_app = ROAD.lock().await; + let (region, location) = match readable_app.filter(uri, method.clone(), headers) { Some(val) => val, None => { return Err(Error::from_string( @@ -93,5 +96,27 @@ pub async fn handle( } } - forward(destination, req, uri, headers, method, body).await + let reg = region.clone(); + let loc = location.clone(); + let end = destination.clone(); + + match forward(&end, req, uri, headers, method, body).await { + Ok(resp) => { + tokio::spawn(async move { + let writable_app = &mut ROAD.lock().await; + writable_app.metrics.add_success_request(reg, loc, end); + }); + Ok(resp) + } + Err(err) => { + let message = format!("{:}", err); + tokio::spawn(async move { + let writable_app = &mut ROAD.lock().await; + writable_app + .metrics + .add_faliure_request(reg, loc, end, message); + }); + Err(err) + } + } } diff --git a/src/sideload/overview.rs b/src/sideload/overview.rs index 1f0b0f1..ba2bc4c 100644 --- a/src/sideload/overview.rs +++ b/src/sideload/overview.rs @@ -1,14 +1,81 @@ -use poem_openapi::{param::Query, payload::PlainText, OpenApi}; +use poem_openapi::{payload::Json, ApiResponse, Object, OpenApi}; + +use crate::{ + proxies::{ + config::{Destination, Location}, + metrics::RoadTrace, + }, + ROAD, +}; use super::SideloadApi; +#[derive(ApiResponse)] +enum OverviewResponse { + /// Return the overview data. + #[oai(status = 200)] + Ok(Json), +} + +#[derive(Debug, Object, Clone, PartialEq)] +struct OverviewData { + /// Loaded regions count + #[oai(read_only)] + regions: usize, + /// Loaded locations count + #[oai(read_only)] + locations: usize, + /// Loaded destnations count + #[oai(read_only)] + destinations: usize, + /// Recent requests count + requests_count: u64, + /// Recent requests success count + faliures_count: u64, + /// Recent requests falied count + successes_count: u64, + /// Recent requests success rate + success_rate: f64, + /// Recent successes + recent_successes: Vec, + /// Recent errors + recent_errors: Vec, +} + #[OpenApi] impl SideloadApi { - #[oai(path = "/hello", method = "get")] - async fn index(&self, name: Query>) -> PlainText { - match name.0 { - Some(name) => PlainText(format!("hello, {name}!")), - None => PlainText("hello!".to_string()), - } + #[oai(path = "/", method = "get")] + async fn index(&self) -> OverviewResponse { + let locked_app = ROAD.lock().await; + let regions = locked_app.regions.clone(); + let locations = regions + .iter() + .flat_map(|item| item.locations.clone()) + .collect::>(); + let destinations = locations + .iter() + .flat_map(|item| item.destinations.clone()) + .collect::>(); + OverviewResponse::Ok(Json(OverviewData { + regions: regions.len(), + locations: locations.len(), + destinations: destinations.len(), + requests_count: locked_app.metrics.requests_count, + successes_count: locked_app.metrics.requests_count - locked_app.metrics.failures_count, + faliures_count: locked_app.metrics.failures_count, + success_rate: locked_app.metrics.get_success_rate(), + recent_successes: locked_app + .metrics + .recent_successes + .clone() + .into_iter() + .collect::>(), + recent_errors: locked_app + .metrics + .recent_errors + .clone() + .into_iter() + .collect::>(), + })) } }