Skip to content
This repository has been archived by the owner on May 1, 2023. It is now read-only.

Commit

Permalink
config file exclude; right flow for users bool
Browse files Browse the repository at this point in the history 
Signed-off-by: Joshua Castle <26531652+Kas-tle@users.noreply.github.com>
  • Loading branch information
@Kas-tle
Kas-tle committed Apr 23, 2023
1 parent 8b72646 commit b77981f
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 26 deletions.
9 changes: 8 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,14 @@ Optionally, create a `config.json` file in the root directory of the project. Ot
"photos": true,
"wall": true
},
"files": false
"files": {
"s3": false,
"wiki": false,
"avatars": true,
"profileCovers": true,
"gameBoxes": true,
"userAlbums": true
}
},
"debug": false,
"disableSSL": false
Expand Down
42 changes: 28 additions & 14 deletions index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ async function main(): Promise<void> {
statusMessage(MessageType.Critical, 'Users already scraped, skipping user tag scraping...');
} else {
statusMessage(MessageType.Info, 'Scraping users...');
await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, sessionID, siteAuth, config.apiKey, config.disabledModules.users);
await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, config.apiKey, config.disabledModules.users);
await insertRow(database, 'scrapers', 'users', true);
await isModuleScraped(database, 'user_data') ? {} : getAdditionalUserData(config.domain, sessionID, siteAuth, database, config.disabledModules.users);
await insertRow(database, 'scrapers', 'user_data', true);
Expand All @@ -172,7 +172,7 @@ async function main(): Promise<void> {
}

// Get files
if (config.disabledModules?.files) {
if (config.disabledModules?.files === true) {
statusMessage(MessageType.Critical, 'Files module disabled, skipping file scraping...');
} else if (
await isModuleScraped(database, 's3_files') &&
Expand All @@ -185,18 +185,32 @@ async function main(): Promise<void> {
statusMessage(MessageType.Critical, 'Files already scraped, skipping file scraping...');
} else {
statusMessage(MessageType.Info, 'Scraping files...');
await isModuleScraped(database, 's3_files') ? {} : await getS3Files(config.domain, database, siteAuth, siteID);
await insertRow(database, 'scrapers', 's3_files', true);
await isModuleScraped(database, 'wiki_files') ? {} :await getWikiFiles(database);
await insertRow(database, 'scrapers', 'wiki_files', true);
await isModuleScraped(database, 'avatar_files') ? {} :await getAvatarFiles(database, siteID);
await insertRow(database, 'scrapers', 'avatar_files', true);
await isModuleScraped(database, 'profile_cover_files') ? {} :await getProfileCoverFiles(database);
await insertRow(database, 'scrapers', 'profile_cover_files', true);
await isModuleScraped(database, 'game_box_files') ? {} :await getGameBoxFiles(database);
await insertRow(database, 'scrapers', 'game_box_files', true);
await isModuleScraped(database, 'user_album_files') ? {} :await getUserAlbumFiles(database);
await insertRow(database, 'scrapers', 'user_album_files', true);
const disabledFileModules = config.disabledModules?.files;
if (!await isModuleScraped(database, 'users') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.s3) : true)) {
await getS3Files(config.domain, database, siteAuth, siteID);
await insertRow(database, 'scrapers', 's3_files', true);
}
if (!await isModuleScraped(database, 'wiki_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.wiki) : true)) {
await getWikiFiles(database);
await insertRow(database, 'scrapers', 'wiki_files', true);
}
if (!await isModuleScraped(database, 'avatar_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.avatars) : true)) {
await getAvatarFiles(database, siteID);
await insertRow(database, 'scrapers', 'avatar_files', true);
}
if (!await isModuleScraped(database, 'profile_cover_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.profileCovers) : true)) {
await getProfileCoverFiles(database);
await insertRow(database, 'scrapers', 'profile_cover_files', true);
}
if (!await isModuleScraped(database, 'game_box_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.gameBoxes) : true)) {
await getGameBoxFiles(database);
await insertRow(database, 'scrapers', 'game_box_files', true);
}
if (!await isModuleScraped(database, 'user_album_files') && ((typeof disabledFileModules === 'object') ? !(disabledFileModules.userAlbums) : true)) {
await getUserAlbumFiles(database);
await insertRow(database, 'scrapers', 'user_album_files', true);
}

deleteFiles([
'./target/recovery/s3_file_progress.json',
'./target/recovery/wiki_file_progress.json',
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "enjinscraper",
"version": "1.2.3",
"version": "1.2.4",
"description": "Scrapes an Enjin site via the Enjin API",
"repository": "https://github.com/Kas-tle/EnjinScraper.git",
"author": "Joshua Castle <packages@kastle.dev",
Expand Down
16 changes: 8 additions & 8 deletions src/scrapers/users.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s

for (let i = userCount[0]; i < totalUsers; i++) {
// User IPs
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.ips) : false) {
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.ips) : true) {
const userIPsResponse = await throttledGetRequest(domain, `/ajax.php?s=admin_users&cmd=getUserAdditionalData&user_id=${userIDs[i]}`, {
Cookie: `${siteAuth.phpSessID}; ${siteAuth.csrfToken}`,
Referer: `Referer https://${domain}/admin/users`
Expand All @@ -53,7 +53,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s


// Profile.getFullInfo
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.fullinfo) : false) {
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.fullinfo) : true) {
const fullInfoResponse = await enjinRequest<Profile.GetFullInfo>({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getFullInfo', domain);
if (fullInfoResponse.error) {
statusMessage(MessageType.Error, `Error getting full info for user ${userIDs[i]}: ${fullInfoResponse.error.message}`);
Expand Down Expand Up @@ -107,7 +107,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
}

// Profile.getCharacters
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.characters) : false) {
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.characters) : true) {
const charactersResponse = await enjinRequest<Profile.GetCharacters>({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getCharacters', domain);
const { characters } = charactersResponse.result;
const charactersDB = [];
Expand Down Expand Up @@ -135,7 +135,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
}

// Profile.getGames
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.games) : false) {
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.games) : true) {
const gamesResponse = await enjinRequest<Profile.GetGames>({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getGames', domain);
const gamesDB = [];
for (const game of gamesResponse.result.games) {
Expand All @@ -159,7 +159,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
}

// Profile.getPhotos
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.photos) : false) {
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.photos) : true) {
const photosResponse = await enjinRequest<Profile.GetPhotos>({session_id: sessionID, user_id: userIDs[i]}, 'Profile.getPhotos', domain);
const { albums, photos } = photosResponse.result;

Expand Down Expand Up @@ -218,7 +218,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
}

// Profile.getWall
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.wall) : false) {
if ((typeof disabledUserModules === 'object') ? !(disabledUserModules.wall) : true) {

const wallPostsDB = [];
const wallCommentsDB = [];
Expand Down Expand Up @@ -377,9 +377,9 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
removeExitListeners();
}

export async function getUsers(database: Database, domain: string, sessionID: string, siteAuth: SiteAuth, apiKey: string, disabledUserModules: Config["disabledModules"]["users"]) {
export async function getUsers(database: Database, domain: string, apiKey: string, disabledUserModules: Config["disabledModules"]["users"]) {
if(!fileExists('./target/recovery/user_ips.json')) {
const allUserTags = await getAllUserTags(domain, apiKey, (typeof disabledUserModules === 'object') ? disabledUserModules.tags : true);
const allUserTags = await getAllUserTags(domain, apiKey, (typeof disabledUserModules === 'object') ? disabledUserModules.tags : false);
let result: UserAdmin.Get = {};
const userDB: UsersDB[] = [];

Expand Down
18 changes: 16 additions & 2 deletions src/util/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,14 @@ export interface Config {
photos: boolean;
wall: boolean;
};
files: boolean;
files: boolean | {
s3: boolean;
wiki: boolean;
avatars: boolean;
profileCovers: boolean;
gameBoxes: boolean;
userAlbums: boolean;
};
};
debug: boolean;
disableSSL: boolean;
Expand Down Expand Up @@ -80,7 +87,14 @@ const defaultConfig: Config = {
photos: true,
wall: true,
},
files: false,
files: {
s3: false,
wiki: false,
avatars: true,
profileCovers: true,
gameBoxes: true,
userAlbums: true
},
},
debug: true,
disableSSL: false
Expand Down

0 comments on commit b77981f

Please sign in to comment.