From b77981f5ac9c30d0c78de56b1609b491ac0d36fc Mon Sep 17 00:00:00 2001 From: Joshua Castle <26531652+Kas-tle@users.noreply.github.com> Date: Sun, 23 Apr 2023 12:52:40 -0700 Subject: [PATCH] config file exclude; right flow for users bool Signed-off-by: Joshua Castle <26531652+Kas-tle@users.noreply.github.com> --- README.md | 9 ++++++++- index.ts | 42 ++++++++++++++++++++++++++++-------------- package.json | 2 +- src/scrapers/users.ts | 16 ++++++++-------- src/util/config.ts | 18 ++++++++++++++++-- 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c80fc93..ce4a03e 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,14 @@ Optionally, create a `config.json` file in the root directory of the project. Ot "photos": true, "wall": true }, - "files": false + "files": { + "s3": false, + "wiki": false, + "avatars": true, + "profileCovers": true, + "gameBoxes": true, + "userAlbums": true + } }, "debug": false, "disableSSL": false diff --git a/index.ts b/index.ts index 9e6008f..c7306a1 100644 --- a/index.ts +++ b/index.ts @@ -163,7 +163,7 @@ async function main(): Promise { statusMessage(MessageType.Critical, 'Users already scraped, skipping user tag scraping...'); } else { statusMessage(MessageType.Info, 'Scraping users...'); - await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, sessionID, siteAuth, config.apiKey, config.disabledModules.users); + await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, config.apiKey, config.disabledModules.users); await insertRow(database, 'scrapers', 'users', true); await isModuleScraped(database, 'user_data') ? {} : getAdditionalUserData(config.domain, sessionID, siteAuth, database, config.disabledModules.users); await insertRow(database, 'scrapers', 'user_data', true); @@ -172,7 +172,7 @@ async function main(): Promise { } // Get files - if (config.disabledModules?.files) { + if (config.disabledModules?.files === true) { statusMessage(MessageType.Critical, 'Files module disabled, skipping file scraping...'); } else if ( await isModuleScraped(database, 's3_files') && @@ -185,18 +185,32 @@ async function main(): Promise { statusMessage(MessageType.Critical, 'Files already scraped, skipping file scraping...'); } else { statusMessage(MessageType.Info, 'Scraping files...'); - await isModuleScraped(database, 's3_files') ? {} : await getS3Files(config.domain, database, siteAuth, siteID); - await insertRow(database, 'scrapers', 's3_files', true); - await isModuleScraped(database, 'wiki_files') ? {} :await getWikiFiles(database); - await insertRow(database, 'scrapers', 'wiki_files', true); - await isModuleScraped(database, 'avatar_files') ? {} :await getAvatarFiles(database, siteID); - await insertRow(database, 'scrapers', 'avatar_files', true); - await isModuleScraped(database, 'profile_cover_files') ? {} :await getProfileCoverFiles(database); - await insertRow(database, 'scrapers', 'profile_cover_files', true); - await isModuleScraped(database, 'game_box_files') ? {} :await getGameBoxFiles(database); - await insertRow(database, 'scrapers', 'game_box_files', true); - await isModuleScraped(database, 'user_album_files') ? {} :await getUserAlbumFiles(database); - await insertRow(database, 'scrapers', 'user_album_files', true); + const disabledFileModules = config.disabledModules?.files; + if (!await isModuleScraped(database, 'users') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.s3) : true)) { + await getS3Files(config.domain, database, siteAuth, siteID); + await insertRow(database, 'scrapers', 's3_files', true); + } + if (!await isModuleScraped(database, 'wiki_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.wiki) : true)) { + await getWikiFiles(database); + await insertRow(database, 'scrapers', 'wiki_files', true); + } + if (!await isModuleScraped(database, 'avatar_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.avatars) : true)) { + await getAvatarFiles(database, siteID); + await insertRow(database, 'scrapers', 'avatar_files', true); + } + if (!await isModuleScraped(database, 'profile_cover_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.profileCovers) : true)) { + await getProfileCoverFiles(database); + await insertRow(database, 'scrapers', 'profile_cover_files', true); + } + if (!await isModuleScraped(database, 'game_box_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.gameBoxes) : true)) { + await getGameBoxFiles(database); + await insertRow(database, 'scrapers', 'game_box_files', true); + } + if (!await isModuleScraped(database, 'user_album_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.userAlbums) : true)) { + await getUserAlbumFiles(database); + await insertRow(database, 'scrapers', 'user_album_files', true); + } + deleteFiles([ './target/recovery/s3_file_progress.json', './target/recovery/wiki_file_progress.json', diff --git a/package.json b/package.json index cc3ebd6..c3c7115 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "enjinscraper", - "version": "1.2.3", + "version": "1.2.4", "description": "Scrapes an Enjin site via the Enjin API", "repository": "https://github.com/Kas-tle/EnjinScraper.git", "author": "Joshua Castle ({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getFullInfo', domain); if (fullInfoResponse.error) { statusMessage(MessageType.Error, `Error getting full info for user ${userIDs[i]}: ${fullInfoResponse.error.message}`); @@ -107,7 +107,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s } // Profile.getCharacters - if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.characters) : false) { + if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.characters) : true) { const charactersResponse = await enjinRequest({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getCharacters', domain); const { characters } = charactersResponse.result; const charactersDB = []; @@ -135,7 +135,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s } // Profile.getGames - if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.games) : false) { + if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.games) : true) { const gamesResponse = await enjinRequest({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getGames', domain); const gamesDB = []; for (const game of gamesResponse.result.games) { @@ -159,7 +159,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s } // Profile.getPhotos - if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.photos) : false) { + if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.photos) : true) { const photosResponse = await enjinRequest({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getPhotos', domain); const { albums, photos } = photosResponse.result; @@ -218,7 +218,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s } // Profile.getWall - if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.wall) : false) { + if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.wall) : true) { const wallPostsDB = []; const wallCommentsDB = []; @@ -377,9 +377,9 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s removeExitListeners(); } -export async function getUsers(database: Database, domain: string, sessionID: string, siteAuth: SiteAuth, apiKey: string, disabledUserModules: Config["disabledModules"]["users"]) { +export async function getUsers(database: Database, domain: string, apiKey: string, disabledUserModules: Config["disabledModules"]["users"]) { if(!fileExists('./target/recovery/user_ips.json')) { - const allUserTags = await getAllUserTags(domain, apiKey, (typeof disabledUserModules === 'object') ? disabledUserModules.tags : true); + const allUserTags = await getAllUserTags(domain, apiKey, (typeof disabledUserModules === 'object') ? disabledUserModules.tags : false); let result: UserAdmin.Get = {}; const userDB: UsersDB[] = []; diff --git a/src/util/config.ts b/src/util/config.ts index 3df84e0..d74eb88 100644 --- a/src/util/config.ts +++ b/src/util/config.ts @@ -32,7 +32,14 @@ export interface Config { photos: boolean; wall: boolean; }; - files: boolean; + files: boolean | { + s3: boolean; + wiki: boolean; + avatars: boolean; + profileCovers: boolean; + gameBoxes: boolean; + userAlbums: boolean; + }; }; debug: boolean; disableSSL: boolean; @@ -80,7 +87,14 @@ const defaultConfig: Config = { photos: true, wall: true, }, - files: false, + files: { + s3: false, + wiki: false, + avatars: true, + profileCovers: true, + gameBoxes: true, + userAlbums: true + }, }, debug: true, disableSSL: false