Skip to content

Commit

Permalink
feat: Background job that checks the latest updated mod for each avai…
Browse files Browse the repository at this point in the history
…lable game on CurseForge, and warns the CF-team if it was over 3 hours since the last file was updated.
  • Loading branch information
itssimple committed Dec 17, 2023
1 parent f840749 commit a58dc0c
Show file tree
Hide file tree
Showing 6 changed files with 446 additions and 96 deletions.
6 changes: 5 additions & 1 deletion CFLookup/CFLookup.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,13 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.8.6" />
<PackageReference Include="Hangfire.Dashboard.BasicAuthorization" Version="1.0.2" />
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.9.3" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.1.2" />
<PackageReference Include="NSec.Cryptography" Version="22.4.0" />
<PackageReference Include="CurseForge.APIClient" Version="2.1.0" />
<PackageReference Include="StackExchange.Redis" Version="2.7.4" />
<PackageReference Include="StackExchange.Redis" Version="2.7.10" />
</ItemGroup>

<ProjectExtensions><VisualStudio><UserProperties wwwroot_4manifest_1json__JsonSchema="https://json.schemastore.org/web-manifest-combined.json" /></VisualStudio></ProjectExtensions>
Expand Down
4 changes: 2 additions & 2 deletions CFLookup/DiscordController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ public async Task<IActionResult> InteractionsAsync()

var json = await sr.ReadToEndAsync();

var discordAppId = Environment.GetEnvironmentVariable("DISCORD_APP_ID");
var discordAppId = Environment.GetEnvironmentVariable("DISCORD_APP_ID")!;

var publicDiscordKey = Environment.GetEnvironmentVariable("DISCORD_PUBLIC_KEY");
var publicDiscordKey = Environment.GetEnvironmentVariable("DISCORD_PUBLIC_KEY")!;
Request.Headers.TryGetValue("X-Signature-Ed25519", out var signatureValue);
Request.Headers.TryGetValue("X-Signature-Timestamp", out var signatureTimestamp);

Expand Down
142 changes: 142 additions & 0 deletions CFLookup/Jobs/GetLatestUpdatedModPerGame.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
using CFLookup.Models;
using CurseForge.APIClient;
using CurseForge.APIClient.Models.Games;
using CurseForge.APIClient.Models.Mods;
using Hangfire.Server;
using Microsoft.Data.SqlClient;
using Newtonsoft.Json;
using StackExchange.Redis;
using System.Text;

namespace CFLookup.Jobs
{
public class GetLatestUpdatedModPerGame
{
public static async Task RunAsync(PerformContext context)
{
using(var scope = Program.ServiceProvider.CreateScope())
{
var cfClient = scope.ServiceProvider.GetRequiredService<ApiClient>();
var db = scope.ServiceProvider.GetRequiredService<MSSQLDB>();
var _redis = scope.ServiceProvider.GetRequiredService<ConnectionMultiplexer>();

var _db = _redis.GetDatabase(5);

Console.WriteLine("Fetching all games");

var allGames = new List<Game>();
var games = await cfClient.GetGamesAsync();
if(games != null && games.Pagination.ResultCount > 0)
{
allGames.AddRange(games.Data);
var index = 0;
while(games.Pagination.ResultCount > 0)
{
index += games.Pagination.PageSize;
games = await cfClient.GetGamesAsync(index);
allGames.AddRange(games.Data);
}
}

Console.WriteLine($"Fetched {allGames.Count} games");

DateTimeOffset lastUpdatedMod = DateTimeOffset.MinValue;
Mod? latestUpdatedModData = null;
CurseForge.APIClient.Models.Files.File? latestUpdatedFileData = null;

foreach(var game in allGames)
{
Console.WriteLine($"Starting to check for latest updated mod for {game.Name}");

var latestUpdatedMod = await cfClient.SearchModsAsync(game.Id, sortField: ModsSearchSortField.LastUpdated, sortOrder: ModsSearchSortOrder.Descending, pageSize: 1);
if(latestUpdatedMod != null && latestUpdatedMod.Pagination.ResultCount > 0)
{
var mod = latestUpdatedMod.Data.First();
var latestUpdatedFile = mod.LatestFiles.OrderByDescending(f => f.FileDate).First();
Console.WriteLine($"Latest updated mod for {game.Name} is {mod.Name} with {mod.DownloadCount} downloads and the latest file was updated {latestUpdatedFile.FileDate}");
if (lastUpdatedMod < latestUpdatedFile.FileDate)
{
lastUpdatedMod = latestUpdatedFile.FileDate;
latestUpdatedModData = mod;
latestUpdatedFileData = latestUpdatedFile;
}

var existingGame = await db.ExecuteSingleRowAsync<FileProcessingStatus>(
"SELECT * FROM fileProcessingStatus WHERE gameId = @gameId",
new SqlParameter("@gameId", game.Id)
);

if(existingGame == null)
{
// New game, insert it
await db.ExecuteNonQueryAsync(
"INSERT INTO fileProcessingStatus (last_updated_utc, gameId, modId, fileId) VALUES (@last_updated_utc, @gameId, @modId, @fileId)",
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate),
new SqlParameter("@gameId", game.Id),
new SqlParameter("@modId", mod.Id),
new SqlParameter("@fileId", latestUpdatedFile.Id)
);
}
else
{
// Existing game, update it
await db.ExecuteNonQueryAsync(
"UPDATE fileProcessingStatus SET last_updated_utc = @last_updated_utc, modId = @modId, fileId = @fileId WHERE gameId = @gameId",
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate),
new SqlParameter("@modId", mod.Id),
new SqlParameter("@fileId", latestUpdatedFile.Id),
new SqlParameter("@gameId", game.Id)
);
}

}
else
{
Console.WriteLine($"No mods found for {game.Name}");
}
}

Console.WriteLine($"Last updated mod was updated {lastUpdatedMod}");

if(lastUpdatedMod < DateTimeOffset.UtcNow.AddHours(-3) && latestUpdatedModData != null && latestUpdatedFileData != null)
{
Console.WriteLine("No mods were updated in the last 3 hours, file processing might be down.");

var warned = await _db.StringGetAsync("cf-file-processing-warning");

if(warned.HasValue && warned == "true")
{
Console.WriteLine("Already warned about this, skipping.");
return;
}

var httpClient = scope.ServiceProvider.GetRequiredService<IHttpClientFactory>().CreateClient();
var discordWebhook = Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.Machine) ??
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.User) ??
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.Process) ??
string.Empty;

if(!string.IsNullOrWhiteSpace(discordWebhook))
{
var message = @$"No mods were updated in the last 3 hours, file processing might be down.
Last updated mod was updated {lastUpdatedMod}, and it was {latestUpdatedModData.Name}
(ProjectID: {latestUpdatedModData.Id}, FileId: {latestUpdatedFileData.Id})
https://cflookup.com/{latestUpdatedModData.Id}";
var payload = new
{
content = message,
flags = 4
};

var json = JsonConvert.SerializeObject(payload);
var content = new StringContent(json, Encoding.UTF8, "application/json");
await httpClient.PostAsync(discordWebhook, content);
}

await _db.StringSetAsync("cf-file-processing-warning", "true", TimeSpan.FromHours(1));
return;
}
}
}
}
}
102 changes: 102 additions & 0 deletions CFLookup/MSSQLDB.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
using Microsoft.Data.SqlClient;
using System.Data;

namespace CFLookup
{
public class MSSQLDB : IDisposable
{
private readonly SqlConnection _connection;

public MSSQLDB(SqlConnection connection)
{
_connection = connection;
}

public async Task<DataTable> ExecuteDataTableAsync(string sql, params SqlParameter[] parameters)
{
await EnsureConnected();
var command = GetCommandWithParams(sql, parameters);

using (var da = new SqlDataAdapter(command))
{
DataTable dt = new DataTable();

da.Fill(dt);

return dt;
}
}

public async Task<List<T>> ExecuteListAsync<T>(string sql, params SqlParameter[] parameters)
{
var dt = await ExecuteDataTableAsync(sql, parameters);

List<T> items = new List<T>();

foreach (DataRow row in dt.Rows)
{
T item = (T)Activator.CreateInstance(typeof(T), row);
items.Add(item);
}

return items;
}

public async Task<T> ExecuteSingleRowAsync<T>(string sql, params SqlParameter[] parameters)
{
var rows = await ExecuteListAsync<T>(sql, parameters);

return rows.FirstOrDefault();
}

public async Task<int> ExecuteNonQueryAsync(string sql, params SqlParameter[] parameters)
{
await EnsureConnected();
var command = GetCommandWithParams(sql, parameters);

return await command.ExecuteNonQueryAsync();
}

public async Task<T> ExecuteScalarAsync<T>(string sql, params SqlParameter[] parameters)
{
await EnsureConnected();
var command = GetCommandWithParams(sql, parameters);

var retValue = await command.ExecuteScalarAsync();

if (retValue is T)
return (T)retValue;

return default;
}

private SqlCommand GetCommandWithParams(string sql, SqlParameter[] parameters)
{
var command = _connection.CreateCommand();
command.CommandTimeout = 300;
command.CommandText = sql;
command.Parameters.AddRange(parameters);

return command;
}

private async Task EnsureConnected()
{
if (_connection.State == ConnectionState.Closed)
{
await _connection.OpenAsync();
}
}

public void Dispose()
{
_connection?.Close();
_connection?.Dispose();
}

public void Dispose(bool isDisposing)
{
if (isDisposing) Dispose();
}
}
}
24 changes: 24 additions & 0 deletions CFLookup/Models/FileProcessingStatus.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
using System.Data;

namespace CFLookup.Models
{
public class FileProcessingStatus
{
public Guid FileProcessingStatusId { get; set; }
public DateTimeOffset Created_UTC { get; set; }
public DateTimeOffset Last_Updated_UTC { get; set; }
public int GameId { get; set; }
public int ModId { get; set; }
public int FileId { get; set; }

public FileProcessingStatus(DataRow row)
{
FileProcessingStatusId = (Guid)row["fileProcessingStatusId"];
Created_UTC = (DateTimeOffset)row["created_utc"];
Last_Updated_UTC = (DateTimeOffset)row["last_updated_utc"];
GameId = (int)row["gameId"];
ModId = (int)row["modId"];
FileId = (int)row["fileId"];
}
}
}
Loading

0 comments on commit a58dc0c

Please sign in to comment.