From 48c04b58723f0d7f4b721baf5fd1c0e39c53e6cb Mon Sep 17 00:00:00 2001 From: Florian Schmidt Date: Wed, 4 Sep 2024 15:25:25 +0200 Subject: [PATCH] add /openmensalist endpoint (optional) gated behind env 'OPENMENSA=y', only returns mensen that contain entries --- Cargo.lock | 216 +++++++++++++++++++++++++++++++++++-- Cargo.toml | 4 +- src/constants.rs | 9 +- src/db_operations.rs | 6 +- src/main.rs | 7 ++ src/openmensa_funcs.rs | 84 +++++++++++++++ src/routes.rs | 3 +- src/services.rs | 2 +- src/stuwe_request_funcs.rs | 8 +- src/types.rs | 6 +- 10 files changed, 319 insertions(+), 26 deletions(-) create mode 100644 src/openmensa_funcs.rs diff --git a/Cargo.lock b/Cargo.lock index 43c98a4..661f168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,6 +153,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.6.0" @@ -179,9 +185,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.15" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b" dependencies = [ "shlex", ] @@ -329,6 +335,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -336,6 +357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -344,6 +366,40 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + [[package]] name = "futures-task" version = "0.3.30" @@ -356,10 +412,16 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -387,8 +449,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -515,9 +579,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http", @@ -584,6 +648,18 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -698,6 +774,8 @@ dependencies = [ "log", "pretty_env_logger", "reqwest", + "reqwest-middleware", + "reqwest-retry", "rusqlite", "scraper", "serde", @@ -791,6 +869,17 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -798,7 +887,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -809,7 +912,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.3", "smallvec", "windows-targets", ] @@ -1069,13 +1172,22 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags", + "bitflags 2.6.0", ] [[package]] @@ -1149,6 +1261,51 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "reqwest-middleware" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "thiserror", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a83df1aaec00176d0fabb65dea13f832d2a446ca99107afc17c5d2d4981221d0" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom", + "http", + "hyper", + "parking_lot 0.11.2", + "reqwest", + "reqwest-middleware", + "retry-policies", + "tokio", + "tracing", + "wasm-timer", +] + +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand", +] + [[package]] name = "ring" version = "0.17.8" @@ -1170,7 +1327,7 @@ version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ - "bitflags", + "bitflags 2.6.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -1271,7 +1428,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cssparser", "derive_more", "fxhash", @@ -1404,7 +1561,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "phf_shared 0.10.0", "precomputed-hash", "serde", @@ -1584,7 +1741,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags", + "bitflags 2.6.0", "bytes", "http", "http-body", @@ -1797,6 +1954,21 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.70" @@ -1816,6 +1988,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -1825,6 +2013,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 00545fc..bfb7a48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ chrono = { version = "0.4.38", features = ["serde"] } axum = { version = "0.7.5", features = ["query"] } serde_json = "1.0.121" scraper = "0.20.0" -reqwest = { version = "0.12.5", features = ["rustls-tls"], default-features = false } +reqwest = { version = "0.12.5", features = ["json", "rustls-tls"], default-features = false } anyhow = "1.0.86" tower-http = { version = "0.5.2", features = ["cors"] } http = "1.1.0" @@ -21,6 +21,8 @@ tokio-cron-scheduler = "0.11.0" pretty_env_logger = "0.5.0" log = "0.4.22" lazy_static = "1.5.0" +reqwest-middleware = "0.3.3" +reqwest-retry = "0.6.1" [profile.release] strip = true diff --git a/src/constants.rs b/src/constants.rs index 21e5dde..8da979d 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1,4 +1,9 @@ use std::{collections::BTreeMap, sync::OnceLock}; -pub static MENSEN_MAP: OnceLock> = OnceLock::new(); -pub static MENSEN_MAP_INV: OnceLock> = OnceLock::new(); +use crate::types::Mensa; + +pub static MENSEN_MAP: OnceLock> = OnceLock::new(); +pub static MENSEN_MAP_INV: OnceLock> = OnceLock::new(); + +pub static OPENMENSA_ALL_MENSEN: OnceLock> = OnceLock::new(); +pub static OPENMENSA_LIVE_MENSEN: OnceLock> = OnceLock::new(); diff --git a/src/db_operations.rs b/src/db_operations.rs index 928255c..32907de 100644 --- a/src/db_operations.rs +++ b/src/db_operations.rs @@ -48,7 +48,7 @@ pub fn init_mensa_id_db() -> rusqlite::Result<()> { Ok(()) } -pub async fn save_meal_to_db(date: &str, mensa: u8, json_text: &str) -> rusqlite::Result<()> { +pub async fn save_meal_to_db(date: &str, mensa: u32, json_text: &str) -> rusqlite::Result<()> { let conn = Connection::open(DB_FILENAME)?; conn.execute( "delete from meals where mensa_id = ?1 and date = ?2", @@ -65,7 +65,7 @@ pub async fn save_meal_to_db(date: &str, mensa: u8, json_text: &str) -> rusqlite Ok(()) } -pub async fn get_meals_from_db(requested_date: NaiveDate, mensa: u8) -> Result> { +pub async fn get_meals_from_db(requested_date: NaiveDate, mensa: u32) -> Result> { let date_str = build_date_string(requested_date); let json_text = get_jsonmeals_from_db(&date_str, mensa).await?; if let Some(json_text) = json_text { @@ -79,7 +79,7 @@ async fn json_to_meal(json_text: &str) -> Result> { Ok(serde_json::from_str(json_text)?) } -pub async fn get_jsonmeals_from_db(date: &str, mensa: u8) -> rusqlite::Result> { +pub async fn get_jsonmeals_from_db(date: &str, mensa: u32) -> rusqlite::Result> { let conn = Connection::open(DB_FILENAME)?; let mut stmt = conn.prepare_cached("select json_text from meals where (mensa_id, date) = (?1, ?2)")?; diff --git a/src/main.rs b/src/main.rs index 680f717..78c64e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,12 @@ use constants::{MENSEN_MAP, MENSEN_MAP_INV}; +use openmensa_funcs::init_openmensa_mensen_with_data; use std::env; use tokio::net::TcpListener; mod constants; mod cronjobs; mod db_operations; +mod openmensa_funcs; mod routes; mod services; mod stuwe_request_funcs; @@ -30,6 +32,11 @@ async fn main() { .unwrap(); init_mensa_id_db().unwrap(); + if env::var_os("OPENMENSA").is_some() { + if let Err(e) = init_openmensa_mensen_with_data().await { + log::error!("OpenMensa list fetch failed: {}", e); + } + } // always update cache on startup match update_cache().await { diff --git a/src/openmensa_funcs.rs b/src/openmensa_funcs.rs new file mode 100644 index 0000000..8f84161 --- /dev/null +++ b/src/openmensa_funcs.rs @@ -0,0 +1,84 @@ +use std::vec; + +use anyhow::Result; +use axum::Json; +use http::StatusCode; +use reqwest::Client; +use reqwest_middleware::ClientBuilder; +use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; +use serde::Deserialize; + +use crate::{ + constants::{OPENMENSA_ALL_MENSEN, OPENMENSA_LIVE_MENSEN}, + types::{Mensa, ResponseError}, +}; + +#[derive(Deserialize)] +struct Day { + closed: bool, +} + +pub async fn init_openmensa_mensen_with_data() -> Result<()> { + if OPENMENSA_ALL_MENSEN.get().is_some() { + log::info!("OpenMensa list already initialized"); + return Ok(()); + } + + log::info!("Getting OpenMensa live mensen list, this might take a while"); + let mut mensen_with_days = vec![]; + + let reqwest_client = Client::builder().build().unwrap(); + let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); + let client = ClientBuilder::new(reqwest_client) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) + .build(); + + let mensen: Vec = client + .get("https://openmensa.org/api/v2/canteens") + .send() + .await? + .error_for_status()? + .json() + .await?; + println!("got {} mensen", mensen.len()); + OPENMENSA_ALL_MENSEN.set(mensen.clone()).unwrap(); + + for (iteration, mensa) in mensen.iter().enumerate() { + if iteration % 50 == 0 { + log::info!("{}%...", (iteration * 100) / mensen.len()); + } + + let days: Vec = client + .get(format!( + "https://openmensa.org/api/v2/canteens/{}/days", + mensa.id + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + + if !days.is_empty() && !days.iter().all(|day| day.closed) { + mensen_with_days.push(mensa.clone()); + } + } + + OPENMENSA_LIVE_MENSEN.set(mensen_with_days).unwrap(); + Ok(()) +} + +pub async fn get_openmensa_list() -> Result>, StatusCode> { + if OPENMENSA_ALL_MENSEN.get().is_none() { + return Err(StatusCode::NOT_FOUND); + } + if let Some(list) = OPENMENSA_LIVE_MENSEN.get() { + Ok(Json(list.clone())) + } else if let Some(list) = OPENMENSA_ALL_MENSEN.get() { + log::warn!("OpenMensa live list not initialized, returning all mensen list"); + Ok(Json(list.clone())) + } else { + log::error!("OpenMensa list not initialized"); + Ok(Json(vec![])) + } +} diff --git a/src/routes.rs b/src/routes.rs index 970720a..82d7b7d 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -2,7 +2,7 @@ use axum::{response::IntoResponse, routing::get, Router}; use http::{header::CONTENT_TYPE, Method}; use tower_http::cors::{Any, CorsLayer}; -use crate::services; +use crate::{openmensa_funcs, services}; pub async fn app() -> Router { let cors = CorsLayer::new() @@ -15,6 +15,7 @@ pub async fn app() -> Router { Router::new() .route("/", get(|| async { "API is reachable".into_response() })) .route("/mensalist", get(services::get_mensa_list)) + .route("/openmensalist", get(openmensa_funcs::get_openmensa_list)) .route("/get_day_at_mensa", get(services::get_day_at_mensa)) .layer(cors) } diff --git a/src/services.rs b/src/services.rs index 539884a..848fe27 100644 --- a/src/services.rs +++ b/src/services.rs @@ -30,7 +30,7 @@ pub async fn get_mensa_list() -> Json> { #[derive(Deserialize, Debug)] pub struct MealsQuery { - pub mensa: u8, + pub mensa: u32, pub date: String, } diff --git a/src/stuwe_request_funcs.rs b/src/stuwe_request_funcs.rs index 96288ac..88dc0dd 100644 --- a/src/stuwe_request_funcs.rs +++ b/src/stuwe_request_funcs.rs @@ -39,7 +39,7 @@ use crate::types::{DataForMensaForDay, MealGroup, MealVariation, SingleMeal}; // println!("{} in {:.2?}", its * strings.len(), now.elapsed()); // } -pub async fn parse_and_save_meals(day: NaiveDate) -> Result> { +pub async fn parse_and_save_meals(day: NaiveDate) -> Result> { let mut today_changed_mensen_ids = vec![]; let date_string = build_date_string(day); @@ -273,7 +273,7 @@ fn extract_mealgroup_from_htmlcontainer(meal_container: ElementRef<'_>) -> Resul Ok(v_meal_groups) } -pub async fn get_mensen() -> Result> { +pub async fn get_mensen() -> Result> { let mut mensen = BTreeMap::new(); // pass invalid date to get empty page (dont need actual data) with all mensa locations @@ -283,7 +283,7 @@ pub async fn get_mensen() -> Result> { let mensa_item_sel = Selector::parse("span").unwrap(); for list_item in document.select(&mensa_list_sel) { if let Some(mensa_id) = list_item.value().attr("data-location") { - if let Ok(mensa_id) = mensa_id.parse::() { + if let Ok(mensa_id) = mensa_id.parse::() { if let Some(mensa_name) = list_item.select(&mensa_item_sel).next() { mensen.insert(mensa_id, mensa_name.inner_html()); } @@ -312,6 +312,6 @@ pub async fn get_mensen() -> Result> { } } -pub fn invert_map(map: &BTreeMap) -> BTreeMap { +pub fn invert_map(map: &BTreeMap) -> BTreeMap { map.iter().map(|(k, v)| (v.clone(), *k)).collect() } diff --git a/src/types.rs b/src/types.rs index 51c87a0..4cbace0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -7,15 +7,15 @@ use http::StatusCode; use serde::{Deserialize, Serialize}; use serde_json::json; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Mensa { - pub id: u8, + pub id: u32, pub name: String, } #[derive(Serialize, Deserialize, Debug)] pub struct DataForMensaForDay { - pub mensa_id: u8, + pub mensa_id: u32, pub meal_groups: Vec, }