Add ability to manually specify user IDs

Signed-off-by: Joshua Castle <26531652+Kas-tle@users.noreply.github.com>
Kas-tle · Apr 28, 2023 · 9634fc5 · 9634fc5
1 parent 62b92d0
commit 9634fc5
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -63,6 +63,7 @@ Optionally, create a `config.json` file in the root directory of the project. Ot
     "manualNewsModuleIDs": [],
     "manualTicketModuleIDs": [],
     "manualWikiModuleIDs": [],
+    "manualUserIDs": [],
     "disabledModules": {
         "html": false,
         "forums": false,

diff --git a/index.ts b/index.ts
@@ -204,7 +204,7 @@ async function main(): Promise<void> {
         statusMessage(MessageType.Critical, 'Users already scraped, skipping user tag scraping...');
     } else {
         statusMessage(MessageType.Info, 'Scraping users...');
-        await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, config.apiKey, config.disabledModules.users);
+        await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, config.apiKey, config.disabledModules.users, config.manualUserIDs ?? []);
         await insertRow(database, 'scrapers', 'users', true);
         await isModuleScraped(database, 'user_data') ? {} : await getAdditionalUserData(config.domain, sessionID, siteAuth, database, config.disabledModules.users, adminMode);
         await insertRow(database, 'scrapers', 'user_data', true);

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "enjinscraper",
-    "version": "1.6.2",
+    "version": "1.6.3",
     "description": "Scrapes an Enjin site via the Enjin API",
     "repository": "https://github.com/Kas-tle/EnjinScraper.git",
     "author": "Joshua Castle <packages@kastle.dev",

diff --git a/src/scrapers/users.ts b/src/scrapers/users.ts
@@ -377,9 +377,10 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
     removeExitListeners();
 }
 
-export async function getUsers(database: Database, domain: string, apiKey: string | null, disabledUserModules: Config["disabledModules"]["users"]) {
+export async function getUsers(database: Database, domain: string, apiKey: string | null, disabledUserModules: Config["disabledModules"]["users"], manualUserIDs: string[]) {
     const allUserTags = await getAllUserTags(domain, apiKey, (typeof disabledUserModules === 'object') ? disabledUserModules.tags : false);
     const userDB: UsersDB[] = [];
+    const userIDs: string[] = [];
 
     if (apiKey !== null) {
         let result: UserAdmin.Get = {};
@@ -431,7 +432,6 @@ export async function getUsers(database: Database, domain: string, apiKey: strin
             }
         } while (Object.keys(result).length > 0);
     } else {
-        const userIDs: string[] = [];
         userIDs.push(...await getColumnUsers(database, 'forums', 'thread_lastpost_user_id'));
         userIDs.push(...await getColumnUsers(database, 'forums'));
         userIDs.push(...await getColumnUsers(database, 'threads', 'thread_user_id'));
@@ -445,9 +445,12 @@ export async function getUsers(database: Database, domain: string, apiKey: strin
         userIDs.push(...await getColumnUsers(database, 'application_responses'));
         userIDs.push(...await getColumnUsers(database, 'application_responses', 'admin_user_id'));
         userIDs.push(...await getColumnUsers(database, 'comments'));
+    }
 
-        const uniqueUserIDs = [...new Set(userIDs)];
+    userIDs.push(...manualUserIDs);
+    const uniqueUserIDs = [...new Set(userIDs)];
 
+    if (uniqueUserIDs.length > 0) {
         for (const userID of uniqueUserIDs) {
             userDB.push([
                 userID,
@@ -470,9 +473,8 @@ export async function getUsers(database: Database, domain: string, apiKey: strin
                 null
             ]);
         }
+        await insertRows(database, 'users', userDB);
     }
-
-    await insertRows(database, 'users', userDB);
 }
 
 async function getColumnUsers(database: Database, table: string, column='user_id'): Promise<string[]> {

diff --git a/src/util/config.ts b/src/util/config.ts
@@ -20,6 +20,7 @@ export interface Config {
     manualNewsModuleIDs?: string[];
     manualTicketModuleIDs?: string[];
     manualWikiModuleIDs?: string[];
+    manualUserIDs?: string[];
     disabledModules: {
         html: boolean;
         forums: boolean;
@@ -75,6 +76,7 @@ const defaultConfig: Config = {
     manualNewsModuleIDs: [],
     manualTicketModuleIDs: [],
     manualWikiModuleIDs: [],
+    manualUserIDs: [],
     disabledModules: {
         html: false,
         forums: false,

diff --git a/src/util/notifier.ts b/src/util/notifier.ts
@@ -17,7 +17,7 @@ export async function startNotifier(database: Database, domain: string, apiKey:
     await new Promise<void>(resolve => process.stdin.once('data', () => resolve()));
 
     const disableUserParams = {ips: true, tags: true, fullinfo: true, characters: true, games: true, photos: true, wall: true};
-    await isModuleScraped(database, 'users') ? {} : await getUsers(database, domain, apiKey, disableUserParams);
+    await isModuleScraped(database, 'users') ? {} : await getUsers(database, domain, apiKey, disableUserParams, []);
 
     const users: { user_id: string, username: string }[] = await new Promise((resolve, reject) => {
         database.all(`SELECT user_id, username FROM users`,