-
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Background job that checks the latest updated mod for each avai…
…lable game on CurseForge, and warns the CF-team if it was over 3 hours since the last file was updated.
- Loading branch information
Showing
6 changed files
with
446 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,142 @@ | ||
using CFLookup.Models; | ||
using CurseForge.APIClient; | ||
using CurseForge.APIClient.Models.Games; | ||
using CurseForge.APIClient.Models.Mods; | ||
using Hangfire.Server; | ||
using Microsoft.Data.SqlClient; | ||
using Newtonsoft.Json; | ||
using StackExchange.Redis; | ||
using System.Text; | ||
|
||
namespace CFLookup.Jobs | ||
{ | ||
public class GetLatestUpdatedModPerGame | ||
{ | ||
public static async Task RunAsync(PerformContext context) | ||
{ | ||
using(var scope = Program.ServiceProvider.CreateScope()) | ||
{ | ||
var cfClient = scope.ServiceProvider.GetRequiredService<ApiClient>(); | ||
var db = scope.ServiceProvider.GetRequiredService<MSSQLDB>(); | ||
var _redis = scope.ServiceProvider.GetRequiredService<ConnectionMultiplexer>(); | ||
|
||
var _db = _redis.GetDatabase(5); | ||
|
||
Console.WriteLine("Fetching all games"); | ||
|
||
var allGames = new List<Game>(); | ||
var games = await cfClient.GetGamesAsync(); | ||
if(games != null && games.Pagination.ResultCount > 0) | ||
{ | ||
allGames.AddRange(games.Data); | ||
var index = 0; | ||
while(games.Pagination.ResultCount > 0) | ||
{ | ||
index += games.Pagination.PageSize; | ||
games = await cfClient.GetGamesAsync(index); | ||
allGames.AddRange(games.Data); | ||
} | ||
} | ||
|
||
Console.WriteLine($"Fetched {allGames.Count} games"); | ||
|
||
DateTimeOffset lastUpdatedMod = DateTimeOffset.MinValue; | ||
Mod? latestUpdatedModData = null; | ||
CurseForge.APIClient.Models.Files.File? latestUpdatedFileData = null; | ||
|
||
foreach(var game in allGames) | ||
{ | ||
Console.WriteLine($"Starting to check for latest updated mod for {game.Name}"); | ||
|
||
var latestUpdatedMod = await cfClient.SearchModsAsync(game.Id, sortField: ModsSearchSortField.LastUpdated, sortOrder: ModsSearchSortOrder.Descending, pageSize: 1); | ||
if(latestUpdatedMod != null && latestUpdatedMod.Pagination.ResultCount > 0) | ||
{ | ||
var mod = latestUpdatedMod.Data.First(); | ||
var latestUpdatedFile = mod.LatestFiles.OrderByDescending(f => f.FileDate).First(); | ||
Console.WriteLine($"Latest updated mod for {game.Name} is {mod.Name} with {mod.DownloadCount} downloads and the latest file was updated {latestUpdatedFile.FileDate}"); | ||
if (lastUpdatedMod < latestUpdatedFile.FileDate) | ||
{ | ||
lastUpdatedMod = latestUpdatedFile.FileDate; | ||
latestUpdatedModData = mod; | ||
latestUpdatedFileData = latestUpdatedFile; | ||
} | ||
|
||
var existingGame = await db.ExecuteSingleRowAsync<FileProcessingStatus>( | ||
"SELECT * FROM fileProcessingStatus WHERE gameId = @gameId", | ||
new SqlParameter("@gameId", game.Id) | ||
); | ||
|
||
if(existingGame == null) | ||
{ | ||
// New game, insert it | ||
await db.ExecuteNonQueryAsync( | ||
"INSERT INTO fileProcessingStatus (last_updated_utc, gameId, modId, fileId) VALUES (@last_updated_utc, @gameId, @modId, @fileId)", | ||
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate), | ||
new SqlParameter("@gameId", game.Id), | ||
new SqlParameter("@modId", mod.Id), | ||
new SqlParameter("@fileId", latestUpdatedFile.Id) | ||
); | ||
} | ||
else | ||
{ | ||
// Existing game, update it | ||
await db.ExecuteNonQueryAsync( | ||
"UPDATE fileProcessingStatus SET last_updated_utc = @last_updated_utc, modId = @modId, fileId = @fileId WHERE gameId = @gameId", | ||
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate), | ||
new SqlParameter("@modId", mod.Id), | ||
new SqlParameter("@fileId", latestUpdatedFile.Id), | ||
new SqlParameter("@gameId", game.Id) | ||
); | ||
} | ||
|
||
} | ||
else | ||
{ | ||
Console.WriteLine($"No mods found for {game.Name}"); | ||
} | ||
} | ||
|
||
Console.WriteLine($"Last updated mod was updated {lastUpdatedMod}"); | ||
|
||
if(lastUpdatedMod < DateTimeOffset.UtcNow.AddHours(-3) && latestUpdatedModData != null && latestUpdatedFileData != null) | ||
{ | ||
Console.WriteLine("No mods were updated in the last 3 hours, file processing might be down."); | ||
|
||
var warned = await _db.StringGetAsync("cf-file-processing-warning"); | ||
|
||
if(warned.HasValue && warned == "true") | ||
{ | ||
Console.WriteLine("Already warned about this, skipping."); | ||
return; | ||
} | ||
|
||
var httpClient = scope.ServiceProvider.GetRequiredService<IHttpClientFactory>().CreateClient(); | ||
var discordWebhook = Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.Machine) ?? | ||
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.User) ?? | ||
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.Process) ?? | ||
string.Empty; | ||
|
||
if(!string.IsNullOrWhiteSpace(discordWebhook)) | ||
{ | ||
var message = @$"No mods were updated in the last 3 hours, file processing might be down. | ||
Last updated mod was updated {lastUpdatedMod}, and it was {latestUpdatedModData.Name} | ||
(ProjectID: {latestUpdatedModData.Id}, FileId: {latestUpdatedFileData.Id}) | ||
https://cflookup.com/{latestUpdatedModData.Id}"; | ||
var payload = new | ||
{ | ||
content = message, | ||
flags = 4 | ||
}; | ||
|
||
var json = JsonConvert.SerializeObject(payload); | ||
var content = new StringContent(json, Encoding.UTF8, "application/json"); | ||
await httpClient.PostAsync(discordWebhook, content); | ||
} | ||
|
||
await _db.StringSetAsync("cf-file-processing-warning", "true", TimeSpan.FromHours(1)); | ||
return; | ||
} | ||
} | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
using Microsoft.Data.SqlClient; | ||
using System.Data; | ||
|
||
namespace CFLookup | ||
{ | ||
public class MSSQLDB : IDisposable | ||
{ | ||
private readonly SqlConnection _connection; | ||
|
||
public MSSQLDB(SqlConnection connection) | ||
{ | ||
_connection = connection; | ||
} | ||
|
||
public async Task<DataTable> ExecuteDataTableAsync(string sql, params SqlParameter[] parameters) | ||
{ | ||
await EnsureConnected(); | ||
var command = GetCommandWithParams(sql, parameters); | ||
|
||
using (var da = new SqlDataAdapter(command)) | ||
{ | ||
DataTable dt = new DataTable(); | ||
|
||
da.Fill(dt); | ||
|
||
return dt; | ||
} | ||
} | ||
|
||
public async Task<List<T>> ExecuteListAsync<T>(string sql, params SqlParameter[] parameters) | ||
{ | ||
var dt = await ExecuteDataTableAsync(sql, parameters); | ||
|
||
List<T> items = new List<T>(); | ||
|
||
foreach (DataRow row in dt.Rows) | ||
{ | ||
T item = (T)Activator.CreateInstance(typeof(T), row); | ||
items.Add(item); | ||
} | ||
|
||
return items; | ||
} | ||
|
||
public async Task<T> ExecuteSingleRowAsync<T>(string sql, params SqlParameter[] parameters) | ||
{ | ||
var rows = await ExecuteListAsync<T>(sql, parameters); | ||
|
||
return rows.FirstOrDefault(); | ||
} | ||
|
||
public async Task<int> ExecuteNonQueryAsync(string sql, params SqlParameter[] parameters) | ||
{ | ||
await EnsureConnected(); | ||
var command = GetCommandWithParams(sql, parameters); | ||
|
||
return await command.ExecuteNonQueryAsync(); | ||
} | ||
|
||
public async Task<T> ExecuteScalarAsync<T>(string sql, params SqlParameter[] parameters) | ||
{ | ||
await EnsureConnected(); | ||
var command = GetCommandWithParams(sql, parameters); | ||
|
||
var retValue = await command.ExecuteScalarAsync(); | ||
|
||
if (retValue is T) | ||
return (T)retValue; | ||
|
||
return default; | ||
} | ||
|
||
private SqlCommand GetCommandWithParams(string sql, SqlParameter[] parameters) | ||
{ | ||
var command = _connection.CreateCommand(); | ||
command.CommandTimeout = 300; | ||
command.CommandText = sql; | ||
command.Parameters.AddRange(parameters); | ||
|
||
return command; | ||
} | ||
|
||
private async Task EnsureConnected() | ||
{ | ||
if (_connection.State == ConnectionState.Closed) | ||
{ | ||
await _connection.OpenAsync(); | ||
} | ||
} | ||
|
||
public void Dispose() | ||
{ | ||
_connection?.Close(); | ||
_connection?.Dispose(); | ||
} | ||
|
||
public void Dispose(bool isDisposing) | ||
{ | ||
if (isDisposing) Dispose(); | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
using System.Data; | ||
|
||
namespace CFLookup.Models | ||
{ | ||
public class FileProcessingStatus | ||
{ | ||
public Guid FileProcessingStatusId { get; set; } | ||
public DateTimeOffset Created_UTC { get; set; } | ||
public DateTimeOffset Last_Updated_UTC { get; set; } | ||
public int GameId { get; set; } | ||
public int ModId { get; set; } | ||
public int FileId { get; set; } | ||
|
||
public FileProcessingStatus(DataRow row) | ||
{ | ||
FileProcessingStatusId = (Guid)row["fileProcessingStatusId"]; | ||
Created_UTC = (DateTimeOffset)row["created_utc"]; | ||
Last_Updated_UTC = (DateTimeOffset)row["last_updated_utc"]; | ||
GameId = (int)row["gameId"]; | ||
ModId = (int)row["modId"]; | ||
FileId = (int)row["fileId"]; | ||
} | ||
} | ||
} |
Oops, something went wrong.