diff --git a/migrations/2024-05-27_initial.sql b/migrations/2024-05-27_initial.sql new file mode 100644 index 0000000..b1fbc99 --- /dev/null +++ b/migrations/2024-05-27_initial.sql @@ -0,0 +1,91 @@ +DROP TABLE IF EXISTS `Badges_Data`; +CREATE TABLE `Badges_Data` ( + `Name` varchar(100) NOT NULL, + `Image_URL` varchar(100) DEFAULT NULL, + PRIMARY KEY (`Name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + +DROP TABLE IF EXISTS `Badge_Name`; +CREATE TABLE `Badge_Name` ( + `Name` varchar(100) NOT NULL, + `User_ID` int(11) NOT NULL, + `Description` varchar(2000) DEFAULT NULL, + `Date_Awarded` datetime DEFAULT NULL, + PRIMARY KEY (`Name`,`User_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + +DROP TABLE IF EXISTS `Medals_Data`; +CREATE TABLE `Medals_Data` ( + `Medal_ID` int(4) NOT NULL, + `Name` varchar(50) DEFAULT NULL, + `Link` varchar(70) DEFAULT NULL, + `Description` varchar(500) DEFAULT NULL, + `Gamemode` varchar(8) DEFAULT NULL, + `Grouping` varchar(30) DEFAULT NULL, + `Instructions` varchar(500) DEFAULT NULL, + `Ordering` int(2) DEFAULT NULL, + `Frequency` float DEFAULT NULL, + `Count_Achieved_By` int(10) DEFAULT NULL, + PRIMARY KEY (`Medal_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + +DROP TABLE IF EXISTS `Rankings_Script_History`; +CREATE TABLE `Rankings_Script_History` ( + `ID` int(8) NOT NULL, + `Type` varchar(30) DEFAULT NULL, + `Time` timestamp NULL DEFAULT NULL, + `Count_Current` int(11) DEFAULT NULL, + `Count_Total` int(11) DEFAULT NULL, + `Elapsed_Seconds` int(20) DEFAULT NULL, + `Elapsed_Last_Update` timestamp NULL DEFAULT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + +DROP TABLE IF EXISTS `Rankings_Users`; +CREATE TABLE `Rankings_Users` ( + `ID` int(11) NOT NULL, + `Accuracy_Catch` decimal(5,2) DEFAULT NULL, + `Accuracy_Mania` decimal(5,2) DEFAULT NULL, + `Accuracy_Standard` decimal(5,2) DEFAULT NULL, + `Accuracy_Stdev` decimal(5,2) DEFAULT NULL, + `Accuracy_Taiko` decimal(5,2) DEFAULT NULL, + `Count_Badges` int(4) DEFAULT NULL, + `Count_Maps_Loved` int(4) DEFAULT NULL, + `Count_Maps_Ranked` int(4) DEFAULT NULL, + `Count_Medals` int(4) DEFAULT NULL, + `Count_Replays_Watched` int(10) DEFAULT NULL, + `Count_Subscribers` int(7) DEFAULT NULL, + `Country_Code` varchar(3) DEFAULT NULL, + `Is_Restricted` int(1) DEFAULT NULL, + `Level_Catch` int(3) DEFAULT NULL, + `Level_Mania` int(3) DEFAULT NULL, + `Level_Standard` int(3) DEFAULT NULL, + `Level_Stdev` int(3) DEFAULT NULL, + `Level_Taiko` int(3) DEFAULT NULL, + `Name` varchar(27) DEFAULT NULL, + `PP_Catch` decimal(8,2) DEFAULT NULL, + `PP_Mania` decimal(8,2) DEFAULT NULL, + `PP_Standard` decimal(8,2) DEFAULT NULL, + `PP_Stdev` decimal(8,2) DEFAULT NULL, + `PP_Taiko` decimal(8,2) DEFAULT NULL, + `PP_Total` decimal(8,2) DEFAULT NULL, + `Rank_Global_Catch` int(20) DEFAULT NULL, + `Rank_Global_Mania` int(20) DEFAULT NULL, + `Rank_Global_Standard` int(20) DEFAULT NULL, + `Rank_Global_Taiko` int(20) DEFAULT NULL, + `Rarest_Medal_Achieved` datetime DEFAULT NULL, + `Rarest_Medal_ID` int(4) DEFAULT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +DROP TABLE IF EXISTS `System_Users`; +CREATE TABLE `System_Users` ( + `User_ID` int(11) NOT NULL, + `Name` varchar(27) DEFAULT NULL, + `Joined_Date` datetime DEFAULT NULL, + PRIMARY KEY (`User_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; \ No newline at end of file diff --git a/src/context/medal.rs b/src/context/medal.rs index 450d869..8a8e59e 100644 --- a/src/context/medal.rs +++ b/src/context/medal.rs @@ -1,15 +1,7 @@ -use std::{ - collections::{HashMap, HashSet}, - fmt::{Formatter, Result as FmtResult}, - string::FromUtf8Error, -}; +use std::{collections::HashMap, string::FromUtf8Error}; use eyre::{Context as _, ContextCompat as _, Result}; use scraper::{Html, Selector}; -use serde::{ - de::{DeserializeSeed, Error as SerdeError, IgnoredAny, MapAccess, SeqAccess, Visitor}, - Deserializer as DeserializerTrait, -}; use crate::{ model::{MedalRarities, OsuUser, ScrapedMedal, ScrapedUser}, @@ -73,59 +65,3 @@ impl Context { .collect() } } - -struct MedalsVisitor; - -impl<'de> Visitor<'de> for MedalsVisitor { - type Value = HashSet; - - fn expecting(&self, f: &mut Formatter<'_>) -> FmtResult { - f.write_str("a list containing objects with a medalid field") - } - - #[inline] - fn visit_seq>(self, mut seq: A) -> Result { - let mut medals = HashSet::with_capacity_and_hasher(300, IntHasher); - - while seq.next_element_seed(MedalId(&mut medals))?.is_some() {} - - Ok(medals) - } -} - -struct MedalId<'m>(&'m mut HashSet); - -impl<'de> DeserializeSeed<'de> for MedalId<'_> { - type Value = (); - - #[inline] - fn deserialize>(self, d: D) -> Result { - d.deserialize_map(self) - } -} - -impl<'de> Visitor<'de> for MedalId<'_> { - type Value = (); - - fn expecting(&self, f: &mut Formatter<'_>) -> FmtResult { - f.write_str("an object with a medalid field") - } - - #[inline] - fn visit_map>(self, mut map: A) -> Result { - let mut medal_id = None; - - while let Some(key) = map.next_key::<&str>()? { - if key == "medalid" { - medal_id = Some(map.next_value()?); - } else { - let _: IgnoredAny = map.next_value()?; - } - } - - let medal_id = medal_id.ok_or_else(|| SerdeError::missing_field("medalid"))?; - self.0.insert(medal_id); - - Ok(()) - } -} diff --git a/src/context/mod.rs b/src/context/mod.rs index e07bc34..3fba7ec 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -132,13 +132,15 @@ impl Context { Err(err) => error!(?err, "Failed to fetch medal ids from DB"), }; - self.handle_rarities_and_ranking(task, users, &medals, &mut db_handles) - .await; - // Store medals if required if task.medals() { - db_handles.push(self.mysql.store_medals(medals)); + // Note that this call needs to happen before storing + // rarities so that the DB table does not deadlock. + self.mysql.store_medals(&medals).await; } + + self.handle_rarities_and_ranking(task, users, &medals, &mut db_handles) + .await; } Err(err) => error!(?err, "Failed to gather medals"), } @@ -205,11 +207,11 @@ impl Context { Err(err) => { error!(?err, "Failed to fetch badges from DB"); - (false, Vec::new()) + (false, Badges::default()) } } } else { - (false, Vec::new()) + (false, Badges::default()) }; if args.debug { @@ -222,9 +224,12 @@ impl Context { let len = user_ids.len(); let mut users = Vec::with_capacity(len); - let mut badges = Badges::with_capacity(10_000); let mut eta = Eta::default(); + let badge_capacity = if check_badges { 10_000 } else { 0 }; + let mut badges_incoming = Badges::with_capacity(badge_capacity); + let mut badge_name_buf = String::new(); + info!("Requesting {len} user(s)..."); let mut progress = Progress::new(len, task); @@ -252,7 +257,7 @@ impl Context { if check_badges { if let OsuUser::Available(ref mut user) = user { for badge in user.badges.iter_mut() { - badges.insert(user_id, badge); + badges_incoming.push(user.user_id, badge, &mut badge_name_buf); } } } @@ -287,25 +292,10 @@ impl Context { } if check_badges { - for (badge_key, badge) in badges.iter_mut() { - let slim_badge = stored_badges - .binary_search_by(|probe| { - probe - .description - .cmp(&badge.description) - .then_with(|| probe.image_url.cmp(&badge_key.image_url)) - }) - .ok() - .and_then(|idx| stored_badges.get(idx)); - - if let Some(slim_badge) = slim_badge { - badge.id = Some(slim_badge.id); - badge.users.extend(&slim_badge.users); - } - } + badges_incoming.merge(stored_badges); } - (users, badges, progress) + (users, badges_incoming, progress) } async fn handle_rarities_and_ranking( diff --git a/src/database/fetch.rs b/src/database/fetch.rs index 3bd8094..0611a91 100644 --- a/src/database/fetch.rs +++ b/src/database/fetch.rs @@ -1,10 +1,14 @@ -use std::{collections::HashSet, ops::DerefMut}; +use std::{ + collections::{HashMap, HashSet}, + ops::DerefMut, +}; -use eyre::{Context as _, Report, Result}; -use futures_util::{StreamExt, TryStreamExt}; +use eyre::{Context as _, Result}; +use futures_util::{future, TryStreamExt}; +use time::OffsetDateTime; use crate::{ - model::{MedalRarities, SlimBadge}, + model::{BadgeDescription, BadgeImageUrl, BadgeName, BadgeOwner, Badges, MedalRarities}, util::IntHasher, }; @@ -22,10 +26,10 @@ impl Database { let query = sqlx::query!( r#" -SELECT - id -FROM - Ranking"# + SELECT + `ID` as id + FROM + Rankings_Users"# ); query @@ -45,14 +49,14 @@ FROM let mut conn = self .acquire() .await - .context("failed to acquire connection to fetch user ids")?; + .context("failed to acquire connection to fetch system user ids")?; let query = sqlx::query!( r#" -SELECT - id -FROM - Members"# + SELECT + `User_ID` as id + FROM + System_Users"# ); query @@ -60,11 +64,10 @@ FROM .map_ok(|row| row.id as u32) .try_collect() .await - .context("failed to fetch all user ids") + .context("failed to fetch system user ids") } - /// The resulting badges will be sorted by their description. - pub async fn fetch_badges(&self) -> Result> { + pub async fn fetch_badges(&self) -> Result { let mut conn = self .acquire() .await @@ -72,45 +75,80 @@ FROM let query = sqlx::query!( r#" -SELECT - id, - description, - users, - image_url -FROM - Badges"# + SELECT + `Name` as name, + `Image_URL` as image_url + FROM + `Badges_Data`"# ); - let mut badges: Vec<_> = query + let names_fut = query .fetch(conn.deref_mut()) - .map(|res| { - let row = res?; - - let users = row - .users - .strip_prefix('[') - .and_then(|suffix| suffix.strip_suffix(']')) - .ok_or(Report::msg("expected square brackets in users string"))? - .split(',') - .map(str::trim) - .map(str::parse) - .collect::, _>>() - .map_err(|_| eyre!("failed to parse id in users string"))?; - - Ok::<_, Report>(SlimBadge { - id: row.id as u32, - description: row.description.into_boxed_str(), - users, - image_url: row.image_url.into_boxed_str(), - }) + .map_ok(|row| { + let name = row.name.into_boxed_str(); + + let image_url = row + .image_url + .map(String::into_boxed_str) + .unwrap_or_default(); + + (BadgeName(name), BadgeImageUrl(image_url)) }) - .try_collect() - .await - .context("failed to fetch all badges")?; + .try_collect(); + + let mut stored = Badges { + names: names_fut.await.context("failed to fetch badges data")?, + descriptions: HashMap::default(), + }; + + let query = sqlx::query!( + r#" + SELECT + `Name` as name, + `User_ID` as user_id, + `Description` as description, + `Date_Awarded` as awarded_at + FROM + `Badge_Name`"# + ); - badges.sort_unstable(); + query + .fetch(conn.deref_mut()) + .try_for_each(|row| { + let owner = BadgeOwner { + user_id: row.user_id as u32, + awarded_at: row + .awarded_at + .map(|datetime| datetime.assume_utc()) + .unwrap_or_else(OffsetDateTime::now_utc), + }; + + let description = row.description.unwrap_or_default(); + + if let Some(names) = stored.descriptions.get_mut(description.as_str()) { + if let Some(owners) = names.get_mut(row.name.as_str()) { + owners.insert(owner); + } else { + names + .entry(BadgeName(row.name.into_boxed_str())) + .or_default() + .insert(owner); + } + } else { + let mut owners = HashSet::default(); + owners.insert(owner); + let mut names = HashMap::default(); + names.insert(BadgeName(row.name.into_boxed_str()), owners); + let description = BadgeDescription(description.into_boxed_str()); + stored.descriptions.insert(description, names); + } + + future::ready(Ok(())) + }) + .await + .context("failed to fetch badge name")?; - Ok(badges) + Ok(stored) } pub async fn fetch_medal_rarities(&self) -> Result { @@ -121,17 +159,23 @@ FROM let query = sqlx::query!( r#" -SELECT - id, - frequency, - count -FROM - MedalRarity"# + SELECT + `Medal_ID` as id, + `Frequency` as frequency, + `Count_Achieved_By` as count + FROM + Medals_Data"# ); query .fetch(conn.deref_mut()) - .map_ok(|row| (row.id as u16, row.count as u32, row.frequency)) + .map_ok(|row| { + ( + row.id as u16, + row.count.unwrap_or(0) as u32, + row.frequency.unwrap_or(0.0), + ) + }) .try_collect() .await .context("failed to fetch all medal rarities") @@ -145,15 +189,15 @@ FROM let query = sqlx::query!( r#" -SELECT - medalid -FROM - Medals"# + SELECT + `Medal_ID` as id + FROM + Medals_Data"# ); query .fetch(conn.deref_mut()) - .map_ok(|row| row.medalid as u16) + .map_ok(|row| row.id as u16) .try_collect() .await .context("failed to fetch all medal ids") diff --git a/src/database/store.rs b/src/database/store.rs index 227c371..94cc6bc 100644 --- a/src/database/store.rs +++ b/src/database/store.rs @@ -4,8 +4,8 @@ use eyre::{Context as _, Result}; use tokio::task::JoinHandle; use crate::model::{ - BadgeEntry, BadgeKey, Badges, Finish, MedalRarities, MedalRarityEntry, Progress, RankingUser, - RankingsIter, ScrapedMedal, + BadgeDescription, BadgeImageUrl, BadgeName, BadgeOwner, Badges, Finish, MedalRarities, + MedalRarityEntry, Progress, RankingUser, RankingsIter, ScrapedMedal, }; use super::Database; @@ -15,9 +15,10 @@ impl Database { let mut conn = self .acquire() .await - .context("failed to acquire connection to update RankingLoopInfo")?; + .context("failed to acquire connection to upsert Rankings_Script_History")?; let Progress { + start, current, total, eta_seconds, @@ -26,16 +27,24 @@ impl Database { let query = sqlx::query!( r#" -UPDATE - RankingLoopInfo -SET - CurrentLoop = ?, - CurrentCount = ?, - TotalCount = ?, - EtaSeconds = ? -LIMIT - 1"#, +INSERT INTO + Rankings_Script_History ( + `ID`, + `Type`, + `Time`, + `Count_Current`, + `Count_Total`, + `Elapsed_Seconds`, + `Elapsed_Last_Update` +) VALUES (?, ?, ?, ?, ?, ?, NOW()) +ON DUPLICATE KEY UPDATE + `Count_Current` = VALUES(`Count_Current`), + `Count_Total` = VALUES(`Count_Total`), + `Elapsed_Seconds` = VALUES(`Elapsed_Seconds`), + `Elapsed_Last_Update` = VALUES(`Elapsed_Last_Update`)"#, + start.unix_timestamp(), task.to_string(), + start, *current as i32, *total as i32, eta_seconds, @@ -44,54 +53,43 @@ LIMIT query .execute(conn.deref_mut()) .await - .context("failed to execute RankingLoopInfo query")?; + .context("failed to execute Rankings_Script_History query")?; Ok(()) } pub async fn store_finish(&self, finish: &Finish) -> Result<()> { - let mut tx = self - .begin() + let mut conn = self + .acquire() .await - .context("failed to begin transaction for RankingLoopHistory")?; + .context("failed to acquire connection to finish Rankings_Script_History")?; let Finish { + id, requested_users, - task, } = finish; - let insert_query = sqlx::query!( - r#" -INSERT INTO RankingLoopHistory (Time, LoopType, Amount) -VALUES - (CURRENT_TIMESTAMP, ?, ?)"#, - task.to_string(), - *requested_users as i32 - ); - - insert_query - .execute(tx.deref_mut()) - .await - .context("failed to execute RankingLoopHistory query")?; - - let update_query = sqlx::query!( + let query = sqlx::query!( r#" UPDATE - RankingLoopInfo + Rankings_Script_History SET - CurrentLoop = "Complete" -LIMIT - 1"# + `Count_Current` = ?, + `Count_Total` = ?, + `Elapsed_Seconds` = ?, + `Elapsed_Last_Update` = NOW() +WHERE + `ID` = ?"#, + *requested_users as i64, + *requested_users as i64, + 0, + id, ); - update_query - .execute(tx.deref_mut()) - .await - .context("failed to execute RankingLoopInfo query")?; - - tx.commit() + query + .execute(conn.deref_mut()) .await - .context("failed to commit finish transaction")?; + .context("failed to execute Rankings_Script_History query")?; Ok(()) } @@ -102,7 +100,7 @@ LIMIT let mut tx = db .begin() .await - .context("failed to begin transaction for Ranking")?; + .context("failed to begin transaction for Rankings_Users")?; for ranking in rankings { let stdev_acc = ranking.std_dev_acc(); @@ -121,10 +119,8 @@ LIMIT badge_count, ranked_maps, loved_maps, - followers, subscribers, replays_watched, - kudosu, restricted, std, tko, @@ -146,106 +142,98 @@ LIMIT let query = sqlx::query!( r#" -INSERT INTO Ranking ( - id, name, total_pp, stdev_pp, standard_pp, - taiko_pp, ctb_pp, mania_pp, medal_count, - rarest_medal, country_code, standard_global, - taiko_global, ctb_global, mania_global, - badge_count, ranked_maps, loved_maps, - subscribers, followers, replays_watched, - rarest_medal_achieved, restricted, - stdev_acc, standard_acc, taiko_acc, - ctb_acc, mania_acc, stdev_level, - standard_level, taiko_level, ctb_level, - mania_level, kudosu, avatar_url -) -VALUES - ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ? - ) ON DUPLICATE KEY -UPDATE - id = VALUES(id), - name = VALUES(name), - total_pp = VALUES(total_pp), - stdev_pp = VALUES(stdev_pp), - standard_pp = VALUES(standard_pp), - taiko_pp = VALUES(taiko_pp), - ctb_pp = VALUES(ctb_pp), - mania_pp = VALUES(mania_pp), - medal_count = VALUES(medal_count), - rarest_medal = VALUES(rarest_medal), - country_code = VALUES(country_code), - standard_global = VALUES(standard_global), - taiko_global = VALUES(taiko_global), - ctb_global = VALUES(ctb_global), - mania_global = VALUES(mania_global), - badge_count = VALUES(badge_count), - ranked_maps = VALUES(ranked_maps), - loved_maps = VALUES(loved_maps), - subscribers = VALUES(subscribers), - followers = VALUES(followers), - replays_watched = VALUES(replays_watched), - rarest_medal_achieved = VALUES(rarest_medal_achieved), - restricted = VALUES(restricted), - stdev_acc = VALUES(stdev_acc), - standard_acc = VALUES(standard_acc), - taiko_acc = VALUES(taiko_acc), - ctb_acc = VALUES(ctb_acc), - mania_acc = VALUES(mania_acc), - stdev_level = VALUES(stdev_level), - standard_level = VALUES(standard_level), - taiko_level = VALUES(taiko_level), - ctb_level = VALUES(ctb_level), - mania_level = VALUES(mania_level), - kudosu = VALUES(kudosu)"#, + INSERT INTO Rankings_Users ( + `ID`, `Accuracy_Catch`, `Accuracy_Mania`, `Accuracy_Standard`, + `Accuracy_Stdev`, `Accuracy_Taiko`, `Count_Badges`, + `Count_Maps_Loved`, `Count_Maps_Ranked`, `Count_Medals`, + `Count_Replays_Watched`, `Count_Subscribers`, `Country_Code`, + `Is_Restricted`, `Level_Catch`, `Level_Mania`, `Level_Standard`, + `Level_Stdev`, `Level_Taiko`, `Name`, `PP_Catch`, `PP_Mania`, + `PP_Standard`, `PP_Stdev`, `PP_Taiko`, `PP_Total`, + `Rank_Global_Catch`, `Rank_Global_Mania`, `Rank_Global_Standard`, + `Rank_Global_Taiko`, `Rarest_Medal_Achieved`, `Rarest_Medal_ID` + ) + VALUES + ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) ON DUPLICATE KEY + UPDATE + `ID` = VALUES(`ID`), + `Accuracy_Catch` = VALUES(`Accuracy_Catch`), + `Accuracy_Mania` = VALUES(`Accuracy_Mania`), + `Accuracy_Standard` = VALUES(`Accuracy_Standard`), + `Accuracy_Stdev` = VALUES(`Accuracy_Stdev`), + `Accuracy_Taiko` = VALUES(`Accuracy_Taiko`), + `Count_Badges` = VALUES(`Count_Badges`), + `Count_Maps_Loved` = VALUES(`Count_Maps_Loved`), + `Count_Maps_Ranked` = VALUES(`Count_Maps_Ranked`), + `Count_Medals` = VALUES(`Count_Medals`), + `Count_Replays_Watched` = VALUES(`Count_Replays_Watched`), + `Count_Subscribers` = VALUES(`Count_Subscribers`), + `Country_Code` = VALUES(`Country_Code`), + `Is_Restricted` = VALUES(`Is_Restricted`), + `Level_Catch` = VALUES(`Level_Catch`), + `Level_Mania` = VALUES(`Level_Mania`), + `Level_Standard` = VALUES(`Level_Standard`), + `Level_Stdev` = VALUES(`Level_Stdev`), + `Level_Taiko` = VALUES(`Level_Taiko`), + `Name` = VALUES(`Name`), + `PP_Catch` = VALUES(`PP_Catch`), + `PP_Mania` = VALUES(`PP_Mania`), + `PP_Standard` = VALUES(`PP_Standard`), + `PP_Stdev` = VALUES(`PP_Stdev`), + `PP_Taiko` = VALUES(`PP_Taiko`), + `PP_Total` = VALUES(`PP_Total`), + `Rank_Global_Catch` = VALUES(`Rank_Global_Catch`), + `Rank_Global_Mania` = VALUES(`Rank_Global_Mania`), + `Rank_Global_Standard` = VALUES(`Rank_Global_Standard`), + `Rank_Global_Taiko` = VALUES(`Rank_Global_Taiko`), + `Rarest_Medal_Achieved` = VALUES(`Rarest_Medal_Achieved`), + `Rarest_Medal_ID` = VALUES(`Rarest_Medal_ID`)"#, id, - name.as_ref(), - total_pp, - stdev_pp, - std.pp, - tko.pp, - ctb.pp, - mna.pp, - medal_count, - rarest_medal_id, - country_code.as_ref(), - std.global_rank.map(NonZeroU32::get), - tko.global_rank.map(NonZeroU32::get), - ctb.global_rank.map(NonZeroU32::get), - mna.global_rank.map(NonZeroU32::get), + ctb_acc, + mna_acc, + std_acc, + stdev_acc, + tko_acc, badge_count, - ranked_maps, loved_maps, - subscribers, - followers, + ranked_maps, + medal_count, replays_watched, - rarest_medal_achieved, + subscribers, + country_code.as_ref(), restricted as u8, - stdev_acc, - std_acc, - tko_acc, - ctb_acc, - mna_acc, - stdev_level, - std.level, - tko.level, ctb.level, mna.level, - kudosu, - 0_i32, // the avatar_url column is no longer needed + std.level, + stdev_level, + tko.level, + name.as_ref(), + ctb.pp, + mna.pp, + std.pp, + stdev_pp, + tko.pp, + total_pp, + ctb.global_rank.map(NonZeroU32::get), + mna.global_rank.map(NonZeroU32::get), + std.global_rank.map(NonZeroU32::get), + tko.global_rank.map(NonZeroU32::get), + rarest_medal_achieved, + rarest_medal_id, ); query .execute(tx.deref_mut()) .await - .context("failed to execute Ranking query")?; + .context("failed to execute Rankings_Users query")?; } tx.commit() .await - .context("failed to commit Ranking transaction")?; + .context("failed to commit Rankings_Users transaction")?; Ok(()) } @@ -264,13 +252,15 @@ UPDATE }) } - #[must_use] - pub fn store_medals(&self, medals: Box<[ScrapedMedal]>) -> JoinHandle<()> { - async fn inner(db: Database, medals: &[ScrapedMedal]) -> Result<()> { + // This method does not return a JoinHandle but is async instead and should + // be called before `Database::store_rarities` so that the table does not + // deadlock. + pub async fn store_medals(&self, medals: &[ScrapedMedal]) { + async fn inner(db: &Database, medals: &[ScrapedMedal]) -> Result<()> { let mut tx = db .begin() .await - .context("failed to begin transaction for Medals")?; + .context("failed to begin transaction for Medals_Data")?; for medal in medals { let ScrapedMedal { @@ -286,27 +276,27 @@ UPDATE let query = sqlx::query!( r#" -INSERT INTO Medals ( - medalid, name, link, description, - restriction, `grouping`, instructions, - ordering -) -VALUES - (?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY -UPDATE - medalid = VALUES(medalid), - name = VALUES(name), - link = VALUES(link), - description = VALUES(description), - restriction = VALUES(restriction), - `grouping` = VALUES(`grouping`), - ordering = VALUES(ordering), - instructions = VALUES(instructions)"#, + INSERT INTO `Medals_Data` ( + `Medal_ID`, `Name`, `Link`, `Description`, + `Gamemode`, `Grouping`, `Instructions`, + `Ordering` + ) + VALUES + (?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY + UPDATE + `Medal_ID` = VALUES(`Medal_ID`), + `Name` = VALUES(`Name`), + `Link` = VALUES(`Link`), + `Description` = VALUES(`Description`), + `Gamemode` = VALUES(`Gamemode`), + `Grouping` = VALUES(`Grouping`), + `Instructions` = VALUES(`Instructions`), + `Ordering` = VALUES(`Ordering`)"#, id, name.as_ref(), icon_url.as_ref(), description.as_ref(), - mode.as_deref().unwrap_or("NULL"), + mode.as_deref(), grouping.as_ref(), instructions.as_deref(), ordering, @@ -315,27 +305,23 @@ UPDATE query .execute(tx.deref_mut()) .await - .context("failed to execute Medals query")?; + .context("failed to execute Medals_Data query")?; } tx.commit() .await - .context("failed to commit Medals transaction")?; + .context("failed to commit Medals_Data transaction")?; Ok(()) } - let db = self.to_owned(); - - tokio::spawn(async move { - let res = inner(db, &medals).await; - let _entered = info_span!("store_medals").entered(); + let res = inner(self, &medals).await; + let _entered = info_span!("store_medals").entered(); - match res { - Ok(_) => info!("Successfully stored {} medals", medals.len()), - Err(err) => error!(?err, "Failed to store medals"), - } - }) + match res { + Ok(_) => info!("Successfully stored {} medals", medals.len()), + Err(err) => error!(?err, "Failed to store medals"), + } } #[must_use] @@ -344,32 +330,32 @@ UPDATE let mut tx = db .begin() .await - .context("failed to begin transaction for MedalRarity")?; + .context("failed to begin transaction for Medals_Data")?; for (medal_id, MedalRarityEntry { count, frequency }) in rarities.iter() { let query = sqlx::query!( r#" -INSERT INTO MedalRarity (id, frequency, count) -VALUES - (?, ?, ?) ON DUPLICATE KEY -UPDATE - id = VALUES(id), - frequency = VALUES(frequency), - count = VALUES(count)"#, - medal_id, +UPDATE + `Medals_Data` +SET + `Frequency` = ?, + `Count_Achieved_By` = ? +WHERE + `Medal_ID` = ?"#, frequency, - count + count, + medal_id, ); query .execute(tx.deref_mut()) .await - .context("failed to execute MedalRarity query")?; + .context("failed to execute Medals_Data query")?; } tx.commit() .await - .context("failed to commit MedalRarity transaction")?; + .context("failed to commit Medals_Data transaction")?; Ok(()) } @@ -393,47 +379,60 @@ UPDATE let mut tx = db .begin() .await - .context("failed to begin transaction for Badges")?; + .context("failed to begin transaction for badges")?; - sqlx::query!("DELETE FROM Badges") + sqlx::query!("DELETE FROM `Badge_Name`") .execute(tx.deref_mut()) .await - .context("failed to delete rows in Badges")?; - - for (key, value) in badges.iter() { - let BadgeKey { image_url } = key; - - let BadgeEntry { - description, - id, - awarded_at, - users, - } = value; - - let name = image_url - .rsplit_once('/') - .and_then(|(_, file)| file.rsplit_once('.')) - .map(|(name, _)| name.replace(['-', '_'], " ")); + .context("failed to delete rows in Badges_Users")?; + + for (BadgeDescription(description), entries) in badges.descriptions.iter() { + for (BadgeName(name), owners) in entries.iter() { + for owner in owners { + let BadgeOwner { + user_id, + awarded_at, + } = owner; + + let query = sqlx::query!( + r#" + INSERT INTO `Badge_Name` ( + `Name`, `User_ID`, `Description`, `Date_Awarded` + ) + VALUES + (?, ?, ?, ?)"#, + name.as_ref(), + user_id, + description.as_ref(), + awarded_at, + ); + + query + .execute(tx.deref_mut()) + .await + .context("failed to execute badge name query")?; + } + } + } + for (BadgeName(name), BadgeImageUrl(image_url)) in badges.names.iter() { let query = sqlx::query!( r#" -INSERT INTO Badges ( - id, name, image_url, description, awarded_at, users -) -VALUES - (?, ?, ?, ?, ?, ?)"#, - id, - name, + INSERT INTO `Badges_Data` ( + `Name`, `Image_URL` + ) + VALUES + (?, ?) + ON DUPLICATE KEY UPDATE + `Name` = `Name`"#, + name.as_ref(), image_url.as_ref(), - description.as_ref(), - awarded_at, - users.to_string(), ); query .execute(tx.deref_mut()) .await - .context("failed to execute Badges query")?; + .context("failed to execute badges data query")?; } tx.commit() diff --git a/src/model/badge.rs b/src/model/badge.rs index be6433c..ffe19e8 100644 --- a/src/model/badge.rs +++ b/src/model/badge.rs @@ -1,10 +1,8 @@ use std::{ + borrow::{Borrow, Cow}, cmp::Ordering, - collections::{ - hash_map::{Iter, IterMut}, - HashMap, HashSet, - }, - fmt::{Display, Formatter, Result as FmtResult}, + collections::{hash_map::Entry, HashMap, HashSet}, + hash::{Hash, Hasher}, mem, }; @@ -13,104 +11,187 @@ use time::OffsetDateTime; use crate::util::IntHasher; -// Different badges may have the same description so we -// use the image url as key instead. -// -// See github issue #1 -#[derive(Eq, PartialEq, Hash)] -pub struct BadgeKey { - pub image_url: Box, +#[derive(PartialEq, Eq, Hash)] +pub struct BadgeName(pub Box); + +impl Borrow for BadgeName { + fn borrow(&self) -> &str { + self.0.as_ref() + } } -pub struct BadgeEntry { - pub description: Box, - pub id: Option, +pub struct BadgeImageUrl(pub Box); + +#[derive(PartialEq, Eq, Hash)] +pub struct BadgeDescription(pub Box); + +impl Borrow for BadgeDescription { + fn borrow(&self) -> &str { + self.0.as_ref() + } +} + +pub struct BadgeOwner { + pub user_id: u32, pub awarded_at: OffsetDateTime, - pub users: BadgeOwners, } -#[derive(Default)] -pub struct Badges { - inner: HashMap, +impl PartialEq for BadgeOwner { + fn eq(&self, other: &Self) -> bool { + self.user_id == other.user_id + } } -impl Badges { - pub fn with_capacity(capacity: usize) -> Self { - let inner = HashMap::with_capacity(capacity); +impl Eq for BadgeOwner {} - Self { inner } +impl Hash for BadgeOwner { + fn hash(&self, state: &mut H) { + self.user_id.hash(state); } +} - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } +type BadgeOwners = HashSet; - pub fn len(&self) -> usize { - self.inner.len() - } +struct PendingImageUrl<'a>(Cow<'a, str>); - pub fn insert(&mut self, user_id: u32, badge: &mut Badge) { - let image_url = if let Some(idx) = badge.image_url.find('?') { - Box::from(&badge.image_url[..idx]) - } else { - mem::take(&mut badge.image_url).into_boxed_str() - }; +impl PendingImageUrl<'_> { + /// Returns `Err` if there is no `/` or `.` after the `/`. + fn name(&self, buf: &mut String) -> Result<(), ()> { + let (name_raw, _) = self + .0 + .rsplit_once('/') + .and_then(|(_, file)| file.rsplit_once('.')) + .ok_or(())?; - let key = BadgeKey { image_url }; + buf.clear(); - let entry = self.inner.entry(key).or_insert_with(|| BadgeEntry { - description: mem::take(&mut badge.description).into_boxed_str(), - awarded_at: badge.awarded_at, - users: BadgeOwners::default(), - id: None, + let chars = name_raw.chars().map(|ch| match ch { + '-' | '_' => ' ', + _ => ch, }); - entry.users.insert(user_id); - } - - pub fn iter(&self) -> Iter<'_, BadgeKey, BadgeEntry> { - self.inner.iter() - } + buf.extend(chars); - pub fn iter_mut(&mut self) -> IterMut<'_, BadgeKey, BadgeEntry> { - self.inner.iter_mut() + Ok(()) } } -pub struct BadgeOwners(HashSet); +impl<'a> From> for BadgeImageUrl { + fn from(pending: PendingImageUrl<'a>) -> Self { + let image_url = match pending.0 { + Cow::Borrowed(image_url) => Box::from(image_url), + Cow::Owned(image_url) => image_url.into_boxed_str(), + }; -impl BadgeOwners { - fn insert(&mut self, user_id: u32) { - self.0.insert(user_id); + Self(image_url) } +} - pub fn extend(&mut self, user_ids: &[u32]) { - self.0.extend(user_ids); - } +#[derive(Default)] +pub struct Badges { + pub names: HashMap, + /// Different badges might have the same description but owners of the same + /// badge don't necessarily have the same description. + pub descriptions: HashMap>, } -impl Default for BadgeOwners { - fn default() -> Self { - Self(HashSet::with_hasher(IntHasher)) +impl Badges { + pub fn with_capacity(capacity: usize) -> Self { + Self { + names: HashMap::with_capacity(capacity), + descriptions: HashMap::with_capacity(capacity), + } } -} -impl Display for BadgeOwners { - #[inline] - fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - f.write_str("[")?; + pub fn push(&mut self, user_id: u32, original: &mut Badge, name_buf: &mut String) { + // Extract the image url. + let image_url = match original.image_url.split_once('?') { + Some((image_url, _)) => PendingImageUrl(Cow::Borrowed(image_url)), + None => PendingImageUrl(Cow::Owned(mem::take(&mut original.image_url))), + }; - let mut iter = self.0.iter(); + // Extract the name from the image url. + if image_url.name(name_buf).is_err() { + warn!( + "Invalid name for badge with image url `{}`", + original.image_url + ); - if let Some(elem) = iter.next() { - Display::fmt(elem, f)?; + return; + } + + // If it's a new name, add it as entry. + if !self.names.contains_key(name_buf.as_str()) { + let name = Box::from(name_buf.as_str()); + self.names + .insert(BadgeName(name), BadgeImageUrl::from(image_url)); + } - for elem in iter { - write!(f, ",{elem}")?; + let owner = BadgeOwner { + user_id, + awarded_at: original.awarded_at, + }; + + if let Some(entries) = self.descriptions.get_mut(original.description.as_str()) { + // The description already has an entry. + if let Some(owners) = entries.get_mut(name_buf.as_str()) { + // The name already has an entry. + owners.insert(owner); + } else { + // Seeing the name for that description for the first time. + // Adding new entry. + let mut owners = HashSet::default(); + owners.insert(owner); + let name = BadgeName(Box::from(name_buf.as_str())); + entries.insert(name, owners); } + } else { + // Seeing that description for the first time. Adding new entry. + let mut owners = HashSet::default(); + owners.insert(owner); + let name = BadgeName(Box::from(name_buf.as_str())); + let mut entries = HashMap::default(); + entries.insert(name, owners); + let description = + BadgeDescription(mem::take(&mut original.description).into_boxed_str()); + self.descriptions.insert(description, entries); } + } + + pub fn len(&self) -> usize { + self.descriptions + .values() + .flat_map(HashMap::values) + .map(HashSet::len) + .sum() + } + + pub fn is_empty(&self) -> bool { + self.descriptions.is_empty() + } - f.write_str("]") + pub fn merge(&mut self, mut other: Self) { + self.names.extend(other.names.drain()); + + for (description, entries) in other.descriptions.drain() { + match self.descriptions.entry(description) { + Entry::Occupied(entry) => { + let this = entry.into_mut(); + + for (name, owners) in entries { + match this.entry(name) { + Entry::Occupied(entry) => entry.into_mut().extend(owners), + Entry::Vacant(entry) => { + entry.insert(owners); + } + } + } + } + Entry::Vacant(entry) => { + entry.insert(entries); + } + } + } } } diff --git a/src/model/mod.rs b/src/model/mod.rs index 795fbb5..7ee205d 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,5 +1,5 @@ pub use self::{ - badge::{BadgeEntry, BadgeKey, Badges, SlimBadge}, + badge::{BadgeDescription, BadgeImageUrl, BadgeName, BadgeOwner, Badges}, progress::{Finish, Progress}, ranking::{RankingUser, RankingsIter}, rarity::{MedalRarities, MedalRarityEntry}, diff --git a/src/model/progress.rs b/src/model/progress.rs index b1b80b6..56404d5 100644 --- a/src/model/progress.rs +++ b/src/model/progress.rs @@ -1,9 +1,12 @@ use serde::Serialize; +use time::OffsetDateTime; use crate::{task::Task, util::Eta}; #[derive(Serialize)] pub struct Progress { + #[serde(skip)] + pub start: OffsetDateTime, pub current: usize, pub total: usize, pub eta_seconds: Option, @@ -14,7 +17,10 @@ impl Progress { pub const INTERVAL: usize = 100; pub fn new(total: usize, task: Task) -> Self { + let start = OffsetDateTime::now_utc(); + Self { + start, task, total, current: 0, @@ -37,16 +43,15 @@ impl Progress { #[derive(Copy, Clone, Serialize)] pub struct Finish { + pub id: i64, pub requested_users: usize, - pub task: Task, } impl From for Finish { - #[inline] fn from(progress: Progress) -> Self { Self { + id: progress.start.unix_timestamp(), requested_users: progress.total, - task: progress.task, } } } diff --git a/src/model/ranking.rs b/src/model/ranking.rs index 1b7a2eb..3fdc645 100644 --- a/src/model/ranking.rs +++ b/src/model/ranking.rs @@ -15,10 +15,8 @@ pub struct RankingUser { pub badge_count: u16, pub ranked_maps: u16, pub loved_maps: u16, - pub followers: u32, pub subscribers: u32, pub replays_watched: u32, - pub kudosu: i32, pub restricted: bool, pub std: RankingMode, pub tko: RankingMode, @@ -65,10 +63,8 @@ impl RankingUser { badge_count: user.badges.len() as u16, ranked_maps: user.maps_ranked, loved_maps: user.maps_loved, - followers: user.followers, subscribers: user.subscribers, replays_watched: user.replays_watched, - kudosu: user.kudosu, restricted: false, std: RankingMode::from(std), tko: RankingMode::from(tko), @@ -88,10 +84,8 @@ impl RankingUser { badge_count: Default::default(), ranked_maps: Default::default(), loved_maps: Default::default(), - followers: Default::default(), subscribers: Default::default(), replays_watched: Default::default(), - kudosu: Default::default(), std: Default::default(), tko: Default::default(), ctb: Default::default(), diff --git a/src/model/user.rs b/src/model/user.rs index 4f7ea7f..f34c25c 100644 --- a/src/model/user.rs +++ b/src/model/user.rs @@ -38,8 +38,6 @@ pub struct UserFull { pub inner: [ModeStats; 4], pub badges: Box<[Badge]>, pub country_code: Box, - pub followers: u32, - pub kudosu: i32, pub maps_ranked: u16, pub maps_loved: u16, pub medals: Box<[MedalCompact]>, @@ -53,8 +51,6 @@ impl UserFull { pub fn new(std: User, tko: User, ctb: User, mna: User) -> Self { let badges = std.badges.unwrap_or_default().into_boxed_slice(); let country_code = std.country_code.into_string().into_boxed_str(); - let followers = std.follower_count.unwrap_or(0); - let kudosu = std.kudosu.total; let maps_ranked = std.ranked_mapset_count.map_or(0, |count| count as u16); let maps_loved = std.loved_mapset_count.map_or(0, |count| count as u16); let medals = std.medals.unwrap_or_default().into_boxed_slice(); @@ -76,8 +72,6 @@ impl UserFull { inner: [std.into(), tko.into(), ctb.into(), mna.into()], badges, country_code, - followers, - kudosu, maps_ranked, maps_loved, medals,