fix module double ask

Signed-off-by: Joshua Castle <26531652+Kas-tle@users.noreply.github.com>
Kas-tle · Apr 25, 2023 · 30769b3 · 30769b3
1 parent 4406e3a
commit 30769b3
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 19 deletions.
diff --git a/index.ts b/index.ts
@@ -31,7 +31,7 @@ async function main(): Promise<void> {
 
     // Login to API and get session ID
     const sessionID = config.sessionID ? config.sessionID : await authenticateAPI(config.domain, config.email, config.password);
-    
+
     // Login to site and get PHPSESSID and csrf_token
     const siteAuth = config.siteAuth ? config.siteAuth : await authenticateSite(config.domain, config.email, config.password);
 
@@ -47,7 +47,7 @@ async function main(): Promise<void> {
     await initializeTables(database);
 
     // Get site data
-    if(await isModuleScraped(database, 'site_data')) {
+    if (await isModuleScraped(database, 'site_data')) {
         statusMessage(MessageType.Critical, 'Site data already scraped, moving on...');
     } else {
         await getSiteData(config.domain, siteAuth, database, siteID);
@@ -193,7 +193,7 @@ async function main(): Promise<void> {
     if (config.disabledModules?.files === true) {
         statusMessage(MessageType.Critical, 'Files module disabled, skipping file scraping...');
     } else if (
-        await isModuleScraped(database, 's3_files') && 
+        await isModuleScraped(database, 's3_files') &&
         await isModuleScraped(database, 'wiki_files') &&
         await isModuleScraped(database, 'avatar_files') &&
         await isModuleScraped(database, 'profile_cover_files') &&
@@ -230,7 +230,7 @@ async function main(): Promise<void> {
         }
 
         deleteFiles([
-            './target/recovery/s3_file_progress.json', 
+            './target/recovery/s3_file_progress.json',
             './target/recovery/wiki_file_progress.json',
             './target/recovery/avatar_file_progress.json',
             './target/recovery/cover_file_progress.json',

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "enjinscraper",
-    "version": "1.5.0",
+    "version": "1.5.1",
     "description": "Scrapes an Enjin site via the Enjin API",
     "repository": "https://github.com/Kas-tle/EnjinScraper.git",
     "author": "Joshua Castle <packages@kastle.dev",

diff --git a/src/util/request.ts b/src/util/request.ts
@@ -7,23 +7,14 @@ import { writeJsonFile } from "./files";
 import { MessageType, statusMessage } from "./console";
 
 let id = 0;
-let protocol: string;
-let retrySeconds = 5;
-let retryTimes = 5;
-
-(async function () {
-    const config = await getConfig();
-    protocol = config.disableSSL ? 'http' : 'https';
-    retrySeconds = config.retrySeconds >= 0 ? config.retrySeconds : 0;
-    retryTimes = config.retryTimes >= 0 ? config.retryTimes : Number.MAX_SAFE_INTEGER;
-    statusMessage(MessageType.Info, `Using ${protocol} protocol`);
-    statusMessage(MessageType.Info, `Retrying after ${retrySeconds} seconds between requests for a mximum of ${retryTimes} requests`);
-})();
-
-const retryTime = retrySeconds * 1000;
 
 export async function enjinRequest<T>(params: Params, method: string, domain: string, inputHeaders: any = {}): Promise<EnjinResponse<T>> {
     const config = await getConfig();
+    const protocol = config.disableSSL ? 'http' : 'https';
+    const retrySeconds = config.retrySeconds >= 0 ? config.retrySeconds : 0;
+    const retryTimes = config.retryTimes >= 0 ? config.retryTimes : Number.MAX_SAFE_INTEGER;
+    const retryTime = retrySeconds * 1000;
+
     let retries = 0;
     while (retries <= retryTimes) {
         const qid = (++id).toString().padStart(7, '0');
@@ -81,6 +72,11 @@ let lastCallTime = 0;
 
 export async function getRequest(domain: string, url: string, headers: any, debugPath = '', overrideDebug = false, responseType: ResponseType | undefined = undefined): Promise<AxiosResponse> {
     const config = await getConfig();
+    const protocol = config.disableSSL ? 'http' : 'https';
+    const retrySeconds = config.retrySeconds >= 0 ? config.retrySeconds : 0;
+    const retryTimes = config.retryTimes >= 0 ? config.retryTimes : Number.MAX_SAFE_INTEGER;
+    const retryTime = retrySeconds * 1000;
+
     let retries = 0;
     while (retries <= retryTimes) {
         const rid = (++id).toString().padStart(7, '0');
@@ -151,6 +147,11 @@ export async function throttledGetRequest(domain: string, url: string, headers:
 
 export async function postRequest(domain: string, url: string, data: any, headers: any, debugPath = ''): Promise<AxiosResponse> {
     const config = await getConfig();
+    const protocol = config.disableSSL ? 'http' : 'https';
+    const retrySeconds = config.retrySeconds >= 0 ? config.retrySeconds : 0;
+    const retryTimes = config.retryTimes >= 0 ? config.retryTimes : Number.MAX_SAFE_INTEGER;
+    const retryTime = retrySeconds * 1000;
+
     let retries = 0;
     while (retries <= retryTimes) {
         const rid = (++id).toString().padStart(7, '0');