From 4fd31df05cf30ace22336d3cca355669e355fb1a Mon Sep 17 00:00:00 2001 From: Louis Murerwa <35416595+louismurerwa@users.noreply.github.com> Date: Fri, 7 Jun 2024 15:22:26 -0700 Subject: [PATCH] Add Logging And Error Handling in Google Drive (#120) * Add Logging And Error Handling in Google Drive * Clean Up --- package-lock.json | 62 ++++++---- .../apps/google-drive/src/loaders/index.ts | 12 +- .../google-drive/src/services/google-drive.ts | 115 ++++++++++-------- .../apps/google-drive/src/services/oauth.ts | 17 ++- .../src/strategies/google-drive.ts | 32 ++++- packages/ocular/core-config-dev.js | 5 +- 6 files changed, 153 insertions(+), 90 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7c8dca76..19fe7da2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16387,6 +16387,10 @@ "defaults": "^1.0.3" } }, + "node_modules/web-connector": { + "resolved": "packages/apps/web-connector", + "link": true + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", @@ -16395,10 +16399,6 @@ "node": ">= 8" } }, - "node_modules/webConnector": { - "resolved": "packages/apps/webconnector", - "link": true - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -17040,8 +17040,7 @@ "@babel/preset-react": "^7.23.3" } }, - "packages/apps/webconnector": { - "name": "webConnector", + "packages/apps/web-connector": { "version": "0.0.0", "dependencies": { "@ocular/ocular": "*", @@ -17056,7 +17055,7 @@ "@babel/preset-react": "^7.23.3" } }, - "packages/apps/webconnector/node_modules/@puppeteer/browsers": { + "packages/apps/web-connector/node_modules/@puppeteer/browsers": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.2.3.tgz", "integrity": "sha512-bJ0UBsk0ESOs6RFcLXOt99a3yTDcOKlzfjad+rhFwdaG1Lu/Wzq58GHYCDTlZ9z6mldf4g+NTb+TXEfe0PpnsQ==", @@ -17077,7 +17076,7 @@ "node": ">=18" } }, - "packages/apps/webconnector/node_modules/chromium-bidi": { + "packages/apps/web-connector/node_modules/chromium-bidi": { "version": "0.5.19", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.5.19.tgz", "integrity": "sha512-UA6zL77b7RYCjJkZBsZ0wlvCTD+jTjllZ8f6wdO4buevXgTZYjV+XLB9CiEa2OuuTGGTLnI7eN9I60YxuALGQg==", @@ -17090,7 +17089,7 @@ "devtools-protocol": "*" } }, - "packages/apps/webconnector/node_modules/cosmiconfig": { + "packages/apps/web-connector/node_modules/cosmiconfig": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", @@ -17115,17 +17114,17 @@ } } }, - "packages/apps/webconnector/node_modules/devtools-protocol": { + "packages/apps/web-connector/node_modules/devtools-protocol": { "version": "0.0.1286932", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1286932.tgz", "integrity": "sha512-wu58HMQll9voDjR4NlPyoDEw1syfzaBNHymMMZ/QOXiHRNluOnDgu9hp1yHOKYoMlxCh4lSSiugLITe6Fvu1eA==" }, - "packages/apps/webconnector/node_modules/emoji-regex": { + "packages/apps/web-connector/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, - "packages/apps/webconnector/node_modules/lru-cache": { + "packages/apps/web-connector/node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", @@ -17136,12 +17135,12 @@ "node": ">=10" } }, - "packages/apps/webconnector/node_modules/mitt": { + "packages/apps/web-connector/node_modules/mitt": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" }, - "packages/apps/webconnector/node_modules/puppeteer": { + "packages/apps/web-connector/node_modules/puppeteer": { "version": "22.10.0", "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.10.0.tgz", "integrity": "sha512-ZOkZd6a6t0BdKcWb0wAYHWQqCfdlN1PPnXOmg/XNrbo6gJhYWFX4qCNb6ahSn8TpAqBqLCoD4Q010F7GwOM7mA==", @@ -17159,7 +17158,7 @@ "node": ">=18" } }, - "packages/apps/webconnector/node_modules/puppeteer-core": { + "packages/apps/web-connector/node_modules/puppeteer-core": { "version": "22.10.0", "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.10.0.tgz", "integrity": "sha512-I54J4Vy4I07UHsgB1QSmuFoF7KNQjJWcvFBPhtY+ezMdBfwgGDr8dzYrJa11aPgP9kxIUHjhktcMmmfJkOAtTw==", @@ -17174,7 +17173,7 @@ "node": ">=18" } }, - "packages/apps/webconnector/node_modules/semver": { + "packages/apps/web-connector/node_modules/semver": { "version": "7.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", @@ -17188,7 +17187,7 @@ "node": ">=10" } }, - "packages/apps/webconnector/node_modules/string-width": { + "packages/apps/web-connector/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", @@ -17201,7 +17200,7 @@ "node": ">=8" } }, - "packages/apps/webconnector/node_modules/tar-fs": { + "packages/apps/web-connector/node_modules/tar-fs": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.5.tgz", "integrity": "sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg==", @@ -17214,7 +17213,7 @@ "bare-path": "^2.1.0" } }, - "packages/apps/webconnector/node_modules/tar-stream": { + "packages/apps/web-connector/node_modules/tar-stream": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", @@ -17224,7 +17223,7 @@ "streamx": "^2.15.0" } }, - "packages/apps/webconnector/node_modules/ws": { + "packages/apps/web-connector/node_modules/ws": { "version": "8.17.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", @@ -17244,12 +17243,12 @@ } } }, - "packages/apps/webconnector/node_modules/yallist": { + "packages/apps/web-connector/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, - "packages/apps/webconnector/node_modules/yargs": { + "packages/apps/web-connector/node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", @@ -17266,7 +17265,7 @@ "node": ">=12" } }, - "packages/apps/webconnector/node_modules/zod": { + "packages/apps/web-connector/node_modules/zod": { "version": "3.22.4", "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", @@ -17274,6 +17273,23 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "packages/apps/webconnector": { + "name": "webConnector", + "version": "0.0.0", + "extraneous": true, + "dependencies": { + "@ocular/ocular": "*", + "@ocular/types": "*", + "@ocular/utils": "*", + "puppeteer": "^22.10.0" + }, + "devDependencies": { + "@babel/cli": "^7.24.1", + "@babel/core": "^7.24.1", + "@babel/preset-env": "^7.24.1", + "@babel/preset-react": "^7.23.3" + } + }, "packages/ocular": { "name": "@ocular/ocular", "version": "1.0.0", diff --git a/packages/apps/google-drive/src/loaders/index.ts b/packages/apps/google-drive/src/loaders/index.ts index ccbcb95d..ea4cf452 100644 --- a/packages/apps/google-drive/src/loaders/index.ts +++ b/packages/apps/google-drive/src/loaders/index.ts @@ -1,11 +1,15 @@ import { AppNameDefinitions, RateLimiterOpts } from "@ocular/types"; import { RateLimiterService } from "@ocular/ocular"; +import { AutoflowAiError, AutoflowAiErrorTypes } from "@ocular/utils"; export default async (container, options) => { try { // Register Rate Limiter For Google Drive if (!options.rate_limiter_opts) { - throw new Error("No options provided for rate limiter"); + throw new AutoflowAiError( + AutoflowAiErrorTypes.INVALID_DATA, + "registerRateLimiter: No options provided for rate limiter for Google Drive" + ); } const rateLimiterOpts: RateLimiterOpts = options.rate_limiter_opts; const rateLimiterService: RateLimiterService = @@ -16,6 +20,10 @@ export default async (container, options) => { rateLimiterOpts.interval ); } catch (err) { - console.log(err); + throw new AutoflowAiError( + AutoflowAiErrorTypes.INVALID_DATA, + "registerRateLimiter: Failed to register rate limiter for Google Drive with error: " + + err.message + ); } }; diff --git a/packages/apps/google-drive/src/services/google-drive.ts b/packages/apps/google-drive/src/services/google-drive.ts index 38f9aafd..2b44f189 100644 --- a/packages/apps/google-drive/src/services/google-drive.ts +++ b/packages/apps/google-drive/src/services/google-drive.ts @@ -18,6 +18,7 @@ import { ConfigModule } from "@ocular/ocular/src/types"; import fs from "fs"; import { google, drive_v3 } from "googleapis"; import { RateLimiterQueue } from "rate-limiter-flexible"; +import { AutoflowAiError, AutoflowAiErrorTypes } from "@ocular/utils"; export default class GoogleDriveService extends TransactionBaseService { protected oauthService_: OAuthService; @@ -44,45 +45,43 @@ export default class GoogleDriveService extends TransactionBaseService { async *getGoogleDriveFiles( org: Organisation ): AsyncGenerator { - this.logger_.info( - `Starting oculation of Google Drive for ${org.id} organisation` - ); - // Check if the OAuth record exists for this App in this Organisation. const oauth = await this.oauthService_.retrieve({ id: org.id, app_name: AppNameDefinitions.GOOGLEDRIVE, }); + if (!oauth) { this.logger_.error( - `No Google Drive OAuth Cred found for ${org.id} organisation` + `getGoogleDriveFiles: No Google Drive OAuth Credential found for ${org.id} organisation` ); - return; + return null; } - // Get the last sync date - this is the time the latest document that was synced from Google Drive. - let last_sync = ""; - if (oauth.last_sync !== null) { - last_sync = oauth.last_sync.toISOString(); - } + try { + // Get the last sync date - this is the time the latest document that was synced from Google Drive. + let last_sync = ""; + if (oauth.last_sync !== null) { + last_sync = oauth.last_sync.toISOString(); + } - // Get the OAuth2Client for the Google Drive App and set the credentials. - const oauth2Client: OAuth2Client = await this.container_[ - `${AppNameDefinitions.GOOGLEDRIVE}Oauth` - ].getOauthCLient(); - oauth2Client.setCredentials({ - access_token: oauth.token, - refresh_token: oauth.refresh_token, - }); - const drive: drive_v3.Drive = await google.drive({ - version: "v3", - auth: oauth2Client, - }); + // Get the OAuth2Client for the Google Drive App and set the credentials. + const oauth2Client: OAuth2Client = await this.container_[ + `${AppNameDefinitions.GOOGLEDRIVE}Oauth` + ].getOauthCLient(); - // Array storing the processed documents - let documents: IndexableDocument[] = []; + oauth2Client.setCredentials({ + access_token: oauth.token, + refresh_token: oauth.refresh_token, + }); - try { + const drive: drive_v3.Drive = await google.drive({ + version: "v3", + auth: oauth2Client, + }); + + // Array storing the processed documents + let documents: IndexableDocument[] = []; // Only Sync Files Modified After Last Sync. let query = "mimeType='application/vnd.google-apps.document'"; if (last_sync !== "") { @@ -106,6 +105,9 @@ export default class GoogleDriveService extends TransactionBaseService { for (const file of data.files) { // Get Each Files Content As Plain Text const content = await this.getGoogleDriveFileContent(file.id, drive); + if (!content) { + continue; + } const doc: IndexableDocument = { id: file.id, organisationId: org.id, @@ -123,6 +125,7 @@ export default class GoogleDriveService extends TransactionBaseService { }; // Batch Documents To Be Yielded To Max 100 At A Time if (documents.length == 100) { + this.logger_.info("getGoogleDriveFiles: Yielding 100 documents"); yield documents; documents = []; } @@ -138,7 +141,9 @@ export default class GoogleDriveService extends TransactionBaseService { // If the error is an unauthorized error, refresh the token and retry the request if (error.response && error.response.status === 401) { // Check if it's an unauthorized error - this.logger_.info(`Refreshing Asana token for ${org.id} organisation`); + this.logger_.info( + `getGoogleDriveFiles: Refreshing Google Drive token for ${org.id} organisation` + ); // Refresh the token const oauthToken = await this.container_[ @@ -150,39 +155,41 @@ export default class GoogleDriveService extends TransactionBaseService { // Retry the request return this.getGoogleDriveFiles(org); - } else { - console.error(error); } - - console.log(error); - console.error("The API returned an error: " + error); + this.logger_.error( + "getGoogleDriveFiles: The API returned an error: " + error.message + ); return null; } - this.logger_.info( - `Finished oculation of Google Drive for ${org.id} organisation` - ); } // Get The Content Of A Google Drive File async getGoogleDriveFileContent(fileId: string, drive: drive_v3.Drive) { - await this.requestQueue_.removeTokens(1, AppNameDefinitions.GOOGLEDRIVE); - return await new Promise((resolve, reject) => { - let data = ""; - drive.files - .export( - { - fileId: fileId, - mimeType: "text/plain", - }, - { responseType: "stream" } - ) - .then((response) => { - response.data - .on("data", (chunk) => (data += chunk)) - .on("end", () => resolve(data)) - .on("error", reject); - }) - .catch(reject); - }); + try { + await this.requestQueue_.removeTokens(1, AppNameDefinitions.GOOGLEDRIVE); + return await new Promise((resolve, reject) => { + let data = ""; + drive.files + .export( + { + fileId: fileId, + mimeType: "text/plain", + }, + { responseType: "stream" } + ) + .then((response) => { + response.data + .on("data", (chunk) => (data += chunk)) + .on("end", () => resolve(data)) + .on("error", reject); + }) + .catch(reject); + }); + } catch (error) { + this.logger_.error( + `getGoogleDriveFileContent: Error fetching content for file ${fileId} with error: ${error.message}` + ); + return null; + } } } diff --git a/packages/apps/google-drive/src/services/oauth.ts b/packages/apps/google-drive/src/services/oauth.ts index 90880cb0..53fc0445 100644 --- a/packages/apps/google-drive/src/services/oauth.ts +++ b/packages/apps/google-drive/src/services/oauth.ts @@ -6,22 +6,27 @@ import { AppCategoryDefinitions, OAuthToken, AppAuthStrategy, + Logger, } from "@ocular/types"; import { config } from "process"; import { OAuth2Client } from "google-auth-library"; import { auth } from "googleapis/build/src/apis/abusiveexperiencereport"; +import { AutoflowAiError, AutoflowAiErrorTypes } from "@ocular/utils"; +import e from "express"; class GoogleDriveOauth extends OauthService { protected client_id_: string; protected client_secret_: string; protected oauth2Client_: OAuth2Client; protected auth_strategy_: AppAuthStrategy; + protected logger_: Logger; constructor(container, options) { super(arguments[0]); this.client_id_ = options.client_id; this.client_secret_ = options.client_secret; this.auth_strategy_ = options.auth_strategy; + this.logger_ = container.logger; this.oauth2Client_ = new google.auth.OAuth2( options.client_id, options.client_secret, @@ -73,8 +78,10 @@ class GoogleDriveOauth extends OauthService { token: accessToken, }; } catch (error) { - console.error(error); - throw error; + this.logger_.error( + "refreshToken: Error refreshing token for Google Drive with error: " + + error.message + ); } } @@ -90,8 +97,10 @@ class GoogleDriveOauth extends OauthService { auth_strategy: AppAuthStrategy.OAUTH_TOKEN_STRATEGY, }; } catch (error) { - console.error(error); - throw error; + this.logger_.error( + "generatingToken: Error generationg token for Google Drive with: " + + error.message + ); } } diff --git a/packages/apps/google-drive/src/strategies/google-drive.ts b/packages/apps/google-drive/src/strategies/google-drive.ts index 4b3ced18..52445c2b 100644 --- a/packages/apps/google-drive/src/strategies/google-drive.ts +++ b/packages/apps/google-drive/src/strategies/google-drive.ts @@ -21,24 +21,50 @@ class GoogleDriveStrategy extends AbstractBatchJobStrategy { protected batchJobService_: BatchJobService; protected googleDriveService_: GoogleDriveService; protected queueService_: QueueService; + protected logger_: Logger; constructor(container) { super(arguments[0]); this.googleDriveService_ = container.googleDriveService; this.batchJobService_ = container.batchJobService; this.queueService_ = container.queueService; + this.logger_ = container.logger; } async processJob(batchJobId: string): Promise { + // Improvements : Check if the status of the previous job scheduled to Occulate Google Drive is still running. + // If it is running, then skip running this job. + // Get Context To Start Job From The BatchJob Context const batchJob = await this.batchJobService_.retrieve(batchJobId); - const stream = await this.googleDriveService_.getGoogleDriveData( - batchJob.context?.org as Organisation + const org = batchJob.context?.org as Organisation; + + // Start Tracking The Activity of The Indexing Process + const oculationGoogleDriveActivity = this.logger_.activity( + `processJob: Oculating Google Drive for organisation: ${org.id} name: ${org.name}` ); + + // Start The Indexing Process + const stream = await this.googleDriveService_.getGoogleDriveData(org); + + // Stream The Data To The Kafka Queue stream.on("data", (documents) => { this.queueService_.sendBatch(APPS_INDEXING_TOPIC, documents); }); + + // Log The End of The Indexing Process + stream.on("end", () => { + this.logger_.success( + oculationGoogleDriveActivity, + `processJob:Starting oculation of Google Drive for ${org.id} organisation` + ); + }); + + // Log The Error of The Indexing Process stream.on("end", () => { - console.log("No more data"); + this.logger_.error( + oculationGoogleDriveActivity, + `processJob:Starting oculation of Google Drive for ${org.id} organisation` + ); }); } diff --git a/packages/ocular/core-config-dev.js b/packages/ocular/core-config-dev.js index 95a941a0..4778e8f1 100644 --- a/packages/ocular/core-config-dev.js +++ b/packages/ocular/core-config-dev.js @@ -3,8 +3,6 @@ const dotenv = require("dotenv"); let ENV_FILE_NAME = ""; switch (process.env.NODE_ENV) { - case "local": - ENV_FILE_NAME = ".env.local"; case "local": ENV_FILE_NAME = ".env.local"; case "production": @@ -20,7 +18,6 @@ switch (process.env.NODE_ENV) { ENV_FILE_NAME = ".env.dev"; break; default: - ENV_FILE_NAME = ".env.local"; ENV_FILE_NAME = ".env.local"; break; } @@ -159,7 +156,7 @@ module.exports = { }, }, { - resolve: `webConnector`, + resolve: `web-connector`, options: { client_id: "FAKE_ID", client_secret: "FAKE_SECRET",