Skip to content

Commit

Permalink
refactor: inex migration (#14)
Browse files Browse the repository at this point in the history 
* initial inex migration

* comment out previous db interactions

* store medals

* use raw timestamp as progress id

* hande finishing progression

* impl user id fetch & rarity store

* finished inex migration
  • Loading branch information
@MaxOhn
MaxOhn authored Jun 9, 2024
1 parent 6745f09 commit 064b647
Show file tree
Hide file tree
Showing 10 changed files with 575 additions and 441 deletions.
91 changes: 91 additions & 0 deletions migrations/2024-05-27_initial.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
DROP TABLE IF EXISTS `Badges_Data`;
CREATE TABLE `Badges_Data` (
`Name` varchar(100) NOT NULL,
`Image_URL` varchar(100) DEFAULT NULL,
PRIMARY KEY (`Name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;


DROP TABLE IF EXISTS `Badge_Name`;
CREATE TABLE `Badge_Name` (
`Name` varchar(100) NOT NULL,
`User_ID` int(11) NOT NULL,
`Description` varchar(2000) DEFAULT NULL,
`Date_Awarded` datetime DEFAULT NULL,
PRIMARY KEY (`Name`,`User_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;


DROP TABLE IF EXISTS `Medals_Data`;
CREATE TABLE `Medals_Data` (
`Medal_ID` int(4) NOT NULL,
`Name` varchar(50) DEFAULT NULL,
`Link` varchar(70) DEFAULT NULL,
`Description` varchar(500) DEFAULT NULL,
`Gamemode` varchar(8) DEFAULT NULL,
`Grouping` varchar(30) DEFAULT NULL,
`Instructions` varchar(500) DEFAULT NULL,
`Ordering` int(2) DEFAULT NULL,
`Frequency` float DEFAULT NULL,
`Count_Achieved_By` int(10) DEFAULT NULL,
PRIMARY KEY (`Medal_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;


DROP TABLE IF EXISTS `Rankings_Script_History`;
CREATE TABLE `Rankings_Script_History` (
`ID` int(8) NOT NULL,
`Type` varchar(30) DEFAULT NULL,
`Time` timestamp NULL DEFAULT NULL,
`Count_Current` int(11) DEFAULT NULL,
`Count_Total` int(11) DEFAULT NULL,
`Elapsed_Seconds` int(20) DEFAULT NULL,
`Elapsed_Last_Update` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;


DROP TABLE IF EXISTS `Rankings_Users`;
CREATE TABLE `Rankings_Users` (
`ID` int(11) NOT NULL,
`Accuracy_Catch` decimal(5,2) DEFAULT NULL,
`Accuracy_Mania` decimal(5,2) DEFAULT NULL,
`Accuracy_Standard` decimal(5,2) DEFAULT NULL,
`Accuracy_Stdev` decimal(5,2) DEFAULT NULL,
`Accuracy_Taiko` decimal(5,2) DEFAULT NULL,
`Count_Badges` int(4) DEFAULT NULL,
`Count_Maps_Loved` int(4) DEFAULT NULL,
`Count_Maps_Ranked` int(4) DEFAULT NULL,
`Count_Medals` int(4) DEFAULT NULL,
`Count_Replays_Watched` int(10) DEFAULT NULL,
`Count_Subscribers` int(7) DEFAULT NULL,
`Country_Code` varchar(3) DEFAULT NULL,
`Is_Restricted` int(1) DEFAULT NULL,
`Level_Catch` int(3) DEFAULT NULL,
`Level_Mania` int(3) DEFAULT NULL,
`Level_Standard` int(3) DEFAULT NULL,
`Level_Stdev` int(3) DEFAULT NULL,
`Level_Taiko` int(3) DEFAULT NULL,
`Name` varchar(27) DEFAULT NULL,
`PP_Catch` decimal(8,2) DEFAULT NULL,
`PP_Mania` decimal(8,2) DEFAULT NULL,
`PP_Standard` decimal(8,2) DEFAULT NULL,
`PP_Stdev` decimal(8,2) DEFAULT NULL,
`PP_Taiko` decimal(8,2) DEFAULT NULL,
`PP_Total` decimal(8,2) DEFAULT NULL,
`Rank_Global_Catch` int(20) DEFAULT NULL,
`Rank_Global_Mania` int(20) DEFAULT NULL,
`Rank_Global_Standard` int(20) DEFAULT NULL,
`Rank_Global_Taiko` int(20) DEFAULT NULL,
`Rarest_Medal_Achieved` datetime DEFAULT NULL,
`Rarest_Medal_ID` int(4) DEFAULT NULL,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

DROP TABLE IF EXISTS `System_Users`;
CREATE TABLE `System_Users` (
`User_ID` int(11) NOT NULL,
`Name` varchar(27) DEFAULT NULL,
`Joined_Date` datetime DEFAULT NULL,
PRIMARY KEY (`User_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
66 changes: 1 addition & 65 deletions src/context/medal.rs
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,7 @@
use std::{
collections::{HashMap, HashSet},
fmt::{Formatter, Result as FmtResult},
string::FromUtf8Error,
};
use std::{collections::HashMap, string::FromUtf8Error};

use eyre::{Context as _, ContextCompat as _, Result};
use scraper::{Html, Selector};
use serde::{
de::{DeserializeSeed, Error as SerdeError, IgnoredAny, MapAccess, SeqAccess, Visitor},
Deserializer as DeserializerTrait,
};

use crate::{
model::{MedalRarities, OsuUser, ScrapedMedal, ScrapedUser},
Expand Down Expand Up @@ -73,59 +65,3 @@ impl Context {
.collect()
}
}

struct MedalsVisitor;

impl<'de> Visitor<'de> for MedalsVisitor {
type Value = HashSet<u16, IntHasher>;

fn expecting(&self, f: &mut Formatter<'_>) -> FmtResult {
f.write_str("a list containing objects with a medalid field")
}

#[inline]
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
let mut medals = HashSet::with_capacity_and_hasher(300, IntHasher);

while seq.next_element_seed(MedalId(&mut medals))?.is_some() {}

Ok(medals)
}
}

struct MedalId<'m>(&'m mut HashSet<u16, IntHasher>);

impl<'de> DeserializeSeed<'de> for MedalId<'_> {
type Value = ();

#[inline]
fn deserialize<D: DeserializerTrait<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_map(self)
}
}

impl<'de> Visitor<'de> for MedalId<'_> {
type Value = ();

fn expecting(&self, f: &mut Formatter<'_>) -> FmtResult {
f.write_str("an object with a medalid field")
}

#[inline]
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
let mut medal_id = None;

while let Some(key) = map.next_key::<&str>()? {
if key == "medalid" {
medal_id = Some(map.next_value()?);
} else {
let _: IgnoredAny = map.next_value()?;
}
}

let medal_id = medal_id.ok_or_else(|| SerdeError::missing_field("medalid"))?;
self.0.insert(medal_id);

Ok(())
}
}
40 changes: 15 additions & 25 deletions src/context/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,13 +132,15 @@ impl Context {
Err(err) => error!(?err, "Failed to fetch medal ids from DB"),
};

self.handle_rarities_and_ranking(task, users, &medals, &mut db_handles)
.await;

// Store medals if required
if task.medals() {
db_handles.push(self.mysql.store_medals(medals));
// Note that this call needs to happen before storing
// rarities so that the DB table does not deadlock.
self.mysql.store_medals(&medals).await;
}

self.handle_rarities_and_ranking(task, users, &medals, &mut db_handles)
.await;
}
Err(err) => error!(?err, "Failed to gather medals"),
}
Expand Down Expand Up @@ -205,11 +207,11 @@ impl Context {
Err(err) => {
error!(?err, "Failed to fetch badges from DB");

(false, Vec::new())
(false, Badges::default())
}
}
} else {
(false, Vec::new())
(false, Badges::default())
};

if args.debug {
Expand All @@ -222,9 +224,12 @@ impl Context {

let len = user_ids.len();
let mut users = Vec::with_capacity(len);
let mut badges = Badges::with_capacity(10_000);
let mut eta = Eta::default();

let badge_capacity = if check_badges { 10_000 } else { 0 };
let mut badges_incoming = Badges::with_capacity(badge_capacity);
let mut badge_name_buf = String::new();

info!("Requesting {len} user(s)...");

let mut progress = Progress::new(len, task);
Expand Down Expand Up @@ -252,7 +257,7 @@ impl Context {
if check_badges {
if let OsuUser::Available(ref mut user) = user {
for badge in user.badges.iter_mut() {
badges.insert(user_id, badge);
badges_incoming.push(user.user_id, badge, &mut badge_name_buf);
}
}
}
Expand Down Expand Up @@ -287,25 +292,10 @@ impl Context {
}

if check_badges {
for (badge_key, badge) in badges.iter_mut() {
let slim_badge = stored_badges
.binary_search_by(|probe| {
probe
.description
.cmp(&badge.description)
.then_with(|| probe.image_url.cmp(&badge_key.image_url))
})
.ok()
.and_then(|idx| stored_badges.get(idx));

if let Some(slim_badge) = slim_badge {
badge.id = Some(slim_badge.id);
badge.users.extend(&slim_badge.users);
}
}
badges_incoming.merge(stored_badges);
}

(users, badges, progress)
(users, badges_incoming, progress)
}

async fn handle_rarities_and_ranking(
Expand Down
Loading

0 comments on commit 064b647

Please sign in to comment.