From a4f6d4a300e2da5ba12d271965272f15634b8437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Fri, 13 Sep 2024 15:58:41 +0200 Subject: [PATCH] Backpopulate missing ranked/submitted dates using new local metadata cache People keep asking why https://github.com/ppy/osu/pull/29553 didn't fix their databases (as stated in the PR, it didn't intend to), so this should do it for them. --- .../LocalCachedBeatmapMetadataSource.cs | 17 ++- .../Database/BackgroundDataStoreProcessor.cs | 104 ++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs index 96817571f6eb..eaa4d8ebfba9 100644 --- a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs +++ b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading.Tasks; using Microsoft.Data.Sqlite; @@ -78,7 +79,7 @@ private bool shouldFetchCache() // cached database exists on disk. && storage.Exists(cache_database_name); - public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? onlineMetadata) + public bool TryLookup(BeatmapInfo beatmapInfo, [NotNullWhen(true)] out OnlineBeatmapMetadata? onlineMetadata) { Debug.Assert(beatmapInfo.BeatmapSet != null); @@ -98,7 +99,7 @@ public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? online try { - using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true)))) + using (var db = getConnection()) { db.Open(); @@ -125,6 +126,9 @@ public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? online return false; } + private SqliteConnection getConnection() => + new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true))); + private void prepareLocalCache() { bool isRefetch = storage.Exists(cache_database_name); @@ -191,6 +195,15 @@ private void prepareLocalCache() }); } + public int GetCacheVersion() + { + using (var connection = getConnection()) + { + connection.Open(); + return getCacheVersion(connection); + } + } + private int getCacheVersion(SqliteConnection connection) { using (var cmd = connection.CreateCommand()) diff --git a/osu.Game/Database/BackgroundDataStoreProcessor.cs b/osu.Game/Database/BackgroundDataStoreProcessor.cs index 16ff766ea4ed..59ef9a3ae158 100644 --- a/osu.Game/Database/BackgroundDataStoreProcessor.cs +++ b/osu.Game/Database/BackgroundDataStoreProcessor.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -11,6 +12,7 @@ using osu.Framework.Bindables; using osu.Framework.Graphics; using osu.Framework.Logging; +using osu.Framework.Platform; using osu.Game.Beatmaps; using osu.Game.Extensions; using osu.Game.Online.API; @@ -61,6 +63,9 @@ public partial class BackgroundDataStoreProcessor : Component [Resolved] private IAPIProvider api { get; set; } = null!; + [Resolved] + private Storage storage { get; set; } = null!; + protected virtual int TimeToSleepDuringGameplay => 30000; protected override void LoadComplete() @@ -78,6 +83,7 @@ protected override void LoadComplete() processScoresWithMissingStatistics(); convertLegacyTotalScoreToStandardised(); upgradeScoreRanks(); + backpopulateMissingSubmissionAndRankDates(); }, TaskCreationOptions.LongRunning).ContinueWith(t => { if (t.Exception?.InnerException is ObjectDisposedException) @@ -443,6 +449,104 @@ private void upgradeScoreRanks() completeNotification(notification, processedCount, scoreIds.Count, failedCount); } + private void backpopulateMissingSubmissionAndRankDates() + { + var localMetadataSource = new LocalCachedBeatmapMetadataSource(storage); + + if (!localMetadataSource.Available) + { + Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is missing."); + return; + } + + try + { + if (localMetadataSource.GetCacheVersion() < 2) + { + Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is too old."); + return; + } + } + catch (Exception ex) + { + Logger.Log($"Error when trying to query version of local metadata cache: {ex}"); + return; + } + + Logger.Log("Querying for beatmap sets that contain missing submission/rank date..."); + + HashSet beatmapSetIds = realmAccess.Run(r => new HashSet( + r.All() + .Where(b => b.StatusInt > 0 && (b.DateRanked == null || b.DateSubmitted == null)) + .AsEnumerable() + .Select(b => b.ID))); + + Logger.Log($"Found {beatmapSetIds.Count} beatmap sets with missing submission/rank date."); + + if (beatmapSetIds.Count == 0) + return; + + var notification = showProgressNotification(beatmapSetIds.Count, "Populating missing submission and rank dates", "beatmap sets now have correct submission and rank dates."); + + int processedCount = 0; + int failedCount = 0; + + foreach (var id in beatmapSetIds) + { + if (notification?.State == ProgressNotificationState.Cancelled) + break; + + updateNotificationProgress(notification, processedCount, beatmapSetIds.Count); + + sleepIfRequired(); + + try + { + // Can't use async overload because we're not on the update thread. + // ReSharper disable once MethodHasAsyncOverload + bool succeeded = realmAccess.Write(r => + { + BeatmapSetInfo beatmapSet = r.Find(id)!; + + // we want any ranked representative of the set. + // the reason for checking ranked status of the difficulty is that it can be locally modified, + // at which point the lookup will fail - but there might still be another unmodified difficulty on which it will work. + if (beatmapSet.Beatmaps.FirstOrDefault(b => b.Status >= BeatmapOnlineStatus.Ranked) is not BeatmapInfo beatmap) + return false; + + bool lookupSucceeded = localMetadataSource.TryLookup(beatmap, out var result); + + if (lookupSucceeded) + { + Debug.Assert(result != null); + beatmapSet.DateRanked = result.DateRanked; + beatmapSet.DateSubmitted = result.DateSubmitted; + return true; + } + + Logger.Log($"Could not find {beatmapSet.GetDisplayString()} in local cache while backpopulating missing submission/rank date"); + return false; + }); + + if (succeeded) + ++processedCount; + else + ++failedCount; + } + catch (ObjectDisposedException) + { + throw; + } + catch (Exception e) + { + Logger.Log($"Failed to update ranked/submitted dates for beatmap set {id}: {e}"); + ++failedCount; + } + } + + completeNotification(notification, processedCount, beatmapSetIds.Count, failedCount); + } + private void updateNotificationProgress(ProgressNotification? notification, int processedCount, int totalCount) { if (notification == null)