Skip to content

Commit

Permalink
Backpopulate missing ranked/submitted dates using new local metadata …
Browse files Browse the repository at this point in the history
…cache

People keep asking why ppy#29553 didn't fix
their databases (as stated in the PR, it didn't intend to), so this
should do it for them.
  • Loading branch information
bdach committed Sep 13, 2024
1 parent 6f14309 commit a4f6d4a
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 2 deletions.
17 changes: 15 additions & 2 deletions osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
Expand Down Expand Up @@ -78,7 +79,7 @@ private bool shouldFetchCache()
// cached database exists on disk.
&& storage.Exists(cache_database_name);

public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? onlineMetadata)
public bool TryLookup(BeatmapInfo beatmapInfo, [NotNullWhen(true)] out OnlineBeatmapMetadata? onlineMetadata)
{
Debug.Assert(beatmapInfo.BeatmapSet != null);

Expand All @@ -98,7 +99,7 @@ public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? online

try
{
using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true))))
using (var db = getConnection())
{
db.Open();

Expand All @@ -125,6 +126,9 @@ public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? online
return false;
}

private SqliteConnection getConnection() =>
new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true)));

private void prepareLocalCache()
{
bool isRefetch = storage.Exists(cache_database_name);
Expand Down Expand Up @@ -191,6 +195,15 @@ private void prepareLocalCache()
});
}

public int GetCacheVersion()
{
using (var connection = getConnection())
{
connection.Open();
return getCacheVersion(connection);
}
}

private int getCacheVersion(SqliteConnection connection)
{
using (var cmd = connection.CreateCommand())
Expand Down
104 changes: 104 additions & 0 deletions osu.Game/Database/BackgroundDataStoreProcessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -11,6 +12,7 @@
using osu.Framework.Bindables;
using osu.Framework.Graphics;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Beatmaps;
using osu.Game.Extensions;
using osu.Game.Online.API;
Expand Down Expand Up @@ -61,6 +63,9 @@ public partial class BackgroundDataStoreProcessor : Component
[Resolved]
private IAPIProvider api { get; set; } = null!;

[Resolved]
private Storage storage { get; set; } = null!;

protected virtual int TimeToSleepDuringGameplay => 30000;

protected override void LoadComplete()
Expand All @@ -78,6 +83,7 @@ protected override void LoadComplete()
processScoresWithMissingStatistics();
convertLegacyTotalScoreToStandardised();
upgradeScoreRanks();
backpopulateMissingSubmissionAndRankDates();
}, TaskCreationOptions.LongRunning).ContinueWith(t =>
{
if (t.Exception?.InnerException is ObjectDisposedException)
Expand Down Expand Up @@ -443,6 +449,104 @@ private void upgradeScoreRanks()
completeNotification(notification, processedCount, scoreIds.Count, failedCount);
}

private void backpopulateMissingSubmissionAndRankDates()
{
var localMetadataSource = new LocalCachedBeatmapMetadataSource(storage);

if (!localMetadataSource.Available)
{
Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is missing.");
return;
}

try
{
if (localMetadataSource.GetCacheVersion() < 2)
{
Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is too old.");
return;
}
}
catch (Exception ex)
{
Logger.Log($"Error when trying to query version of local metadata cache: {ex}");
return;
}

Logger.Log("Querying for beatmap sets that contain missing submission/rank date...");

HashSet<Guid> beatmapSetIds = realmAccess.Run(r => new HashSet<Guid>(
r.All<BeatmapSetInfo>()
.Where(b => b.StatusInt > 0 && (b.DateRanked == null || b.DateSubmitted == null))
.AsEnumerable()
.Select(b => b.ID)));

Logger.Log($"Found {beatmapSetIds.Count} beatmap sets with missing submission/rank date.");

if (beatmapSetIds.Count == 0)
return;

var notification = showProgressNotification(beatmapSetIds.Count, "Populating missing submission and rank dates", "beatmap sets now have correct submission and rank dates.");

int processedCount = 0;
int failedCount = 0;

foreach (var id in beatmapSetIds)
{
if (notification?.State == ProgressNotificationState.Cancelled)
break;

updateNotificationProgress(notification, processedCount, beatmapSetIds.Count);

sleepIfRequired();

try
{
// Can't use async overload because we're not on the update thread.
// ReSharper disable once MethodHasAsyncOverload
bool succeeded = realmAccess.Write(r =>
{
BeatmapSetInfo beatmapSet = r.Find<BeatmapSetInfo>(id)!;
// we want any ranked representative of the set.
// the reason for checking ranked status of the difficulty is that it can be locally modified,
// at which point the lookup will fail - but there might still be another unmodified difficulty on which it will work.
if (beatmapSet.Beatmaps.FirstOrDefault(b => b.Status >= BeatmapOnlineStatus.Ranked) is not BeatmapInfo beatmap)
return false;
bool lookupSucceeded = localMetadataSource.TryLookup(beatmap, out var result);
if (lookupSucceeded)
{
Debug.Assert(result != null);
beatmapSet.DateRanked = result.DateRanked;
beatmapSet.DateSubmitted = result.DateSubmitted;
return true;
}
Logger.Log($"Could not find {beatmapSet.GetDisplayString()} in local cache while backpopulating missing submission/rank date");
return false;
});

if (succeeded)
++processedCount;
else
++failedCount;
}
catch (ObjectDisposedException)
{
throw;
}
catch (Exception e)
{
Logger.Log($"Failed to update ranked/submitted dates for beatmap set {id}: {e}");
++failedCount;
}
}

completeNotification(notification, processedCount, beatmapSetIds.Count, failedCount);
}

private void updateNotificationProgress(ProgressNotification? notification, int processedCount, int totalCount)
{
if (notification == null)
Expand Down

0 comments on commit a4f6d4a

Please sign in to comment.