chore:ES-186 Implement chunked requests for component services

scanoss · Feb 7, 2025 · d30951c · d30951c
1 parent 60b0b52
commit d30951c
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 40 deletions.
diff --git a/src/config/AppConfigDefault.ts b/src/config/AppConfigDefault.ts
@@ -19,6 +19,7 @@ export const AppConfigDefault: IAppConfig = {
   DEFAULT_IP_gRPC: 'api.scanoss.com',
   DEFAULT_PORT_gRPC: 443,
   DEFAULT_MULTIUSER_LOCK_TIMEOUT: 30,
+  DEFAULT_SERVICE_CHUNK_LIMIT: 10,
 
   // feature flags
   FF_ENABLE_COMPONENT_LOGO: true,

diff --git a/src/config/IAppConfig.ts b/src/config/IAppConfig.ts
@@ -10,6 +10,8 @@ export interface IAppConfig {
   ABOUT_MESSAGE: string;
   DEFAULT_SETTING_NAME: string;
 
+  DEFAULT_SERVICE_CHUNK_LIMIT: number;
+
   OSSKB_HOST: string;
   API_URL: string;
   API_KEY: string;

diff --git a/src/main/modules/export/helpers/exportHelper.ts b/src/main/modules/export/helpers/exportHelper.ts
@@ -32,6 +32,8 @@ export function toVulnerabilityExportData(componentVulnerabilities: Array<Compon
     }
   });
   return Array.from(vulnerabilityMapper.values()).flatMap(((v) => {
-    return { ...v, affectedComponents: Array.from(v.components.values()) as Array<AffectedComponent> };
+    const vulnerability = { ...v, affectedComponents: Array.from(v.components.values()) as Array<AffectedComponent> };
+    delete vulnerability.components;
+    return vulnerability;
   }));
 }
diff --git a/src/main/services/LicenseService.ts b/src/main/services/LicenseService.ts
@@ -1,12 +1,12 @@
 import { LicenseDTO, NewLicenseDTO } from '@api/dto';
+import { HttpClient } from 'scanoss';
+import log from 'electron-log';
 import { licenses } from '../../../assets/data/licenses';
 import { modelProvider } from './ModelProvider';
 import { licenseHelper } from '../helpers/LicenseHelper';
-import { HttpClient, HttpProxy } from 'scanoss';
 import { getHttpConfig } from './utils/httpUtil';
 import { userSettingService } from './UserSettingService';
-import { workspace } from '../../main/workspace/Workspace';
-import log from 'electron-log';
+import { workspace } from '../workspace/Workspace';
 import { LicenseObligation } from '../../api/types';
 
 class LicenseService {
@@ -34,29 +34,27 @@ class LicenseService {
   }
 
   public async getLicenseObligations(spdxid: string): Promise<LicenseObligation> {
-
     const project = workspace.getOpenedProjects()[0];
     const {
       DEFAULT_API_INDEX,
-      APIS
+      APIS,
     } = userSettingService.get();
-   
+
     const scanossHttp = new HttpClient(getHttpConfig());
 
-    const URL = project.getApi() ?  project.getApi() : APIS[DEFAULT_API_INDEX].URL;
- 
+    const URL = project.getApi() ? project.getApi() : APIS[DEFAULT_API_INDEX].URL;
+
     const obligationURL = `${URL}/license/obligations/${spdxid}`;
 
     const response = await scanossHttp.get(obligationURL);
 
     if (!response.ok) {
-        log.error("[ License obligations ]", response.statusText);
-        throw new Error(response.statusText);
+      log.error('[ License obligations ]', response.statusText);
+      throw new Error(response.statusText);
     }
-   
-    return await response.json();
+
+    return response.json();
   }
-
 }
 
 export const licenseService = new LicenseService();
diff --git a/src/main/task/cryptography/AddCryptographyTask.ts b/src/main/task/cryptography/AddCryptographyTask.ts
@@ -2,18 +2,17 @@ import log from 'electron-log';
 import {
   CryptographyService,
 } from 'scanoss';
-import fs from 'fs';
 import { ITask } from '../Task';
 import { modelProvider } from '../../services/ModelProvider';
 import { ICryptographyTask } from './ICryptographyTask';
 import { normalizeCryptoAlgorithms } from '../../../shared/adapters/crypto.adapter';
 import { userSettingService } from '../../services/UserSettingService';
+import { AppConfigDefault } from '../../../config/AppConfigDefault';
 
 export class AddCryptographyTask implements ITask<ICryptographyTask, void> {
-
   async run(params: ICryptographyTask): Promise<void> {
-    const {GRPC_PROXY} = userSettingService.get();
-    process.env.grpc_proxy =  GRPC_PROXY ? GRPC_PROXY : '';
+    const { GRPC_PROXY } = userSettingService.get();
+    process.env.grpc_proxy = GRPC_PROXY || '';
 
     try {
       const response = await this.getAlgorithms(params.components, params.token);
@@ -32,11 +31,30 @@ export class AddCryptographyTask implements ITask<ICryptographyTask, void> {
 
   private async getAlgorithms(components: Array<string>, token: string) {
     const crypto = new CryptographyService(token);
-    const reqData = {
-      purlsList: components.map((purl) => ({ purl })),
-    };
-    const response: any = await crypto.getAlgorithms(reqData);
-    return response;
+    const chunks = [];
+    for (let i = 0; i < components.length; i += AppConfigDefault.DEFAULT_SERVICE_CHUNK_LIMIT) {
+      chunks.push(components.slice(i, i + 10));
+    }
+    const promises = chunks.map(async (chunk) => {
+      try {
+        const reqData = {
+          purlsList: chunk.map((purl) => ({ purl })),
+        };
+        return await crypto.getAlgorithms(reqData);
+      } catch (err) {
+        log.error('[ CryptographyTask ] Request failed for purls:', chunk.map((item: any) => item.purl));
+        log.error('Error:', err);
+        return null;
+      }
+    });
+    const results = await Promise.all(promises);
+    return results.reduce((acc:any, curr:any) => {
+      if (!curr) return acc;
+      return {
+        purlsList: [...(acc.purlsList || []), ...(curr.purlsList || [])],
+        status: curr.status,
+      };
+    }, { purlsList: [], status: null });
   }
 
   private adaptToCrypographyEntity(response: any): Array< { purl: string, version: string, algorithms: string } > {

diff --git a/src/main/task/scanner/dependency/DependencyTask.ts b/src/main/task/scanner/dependency/DependencyTask.ts
@@ -9,6 +9,7 @@ import { Scanner } from '../types';
 import { ScannerStage } from '../../../../api/types';
 import { userSettingService } from '../../../services/UserSettingService';
 import { modelProvider } from '../../../services/ModelProvider';
+import AppConfigModule from '../../../../config/AppConfigModule';
 
 export class DependencyTask implements Scanner.IPipelineTask {
   protected project: Project;
@@ -46,10 +47,31 @@ export class DependencyTask implements Scanner.IPipelineTask {
 
       const cfg = new DependencyScannerCfg();
       const { GRPC_PROXY } = userSettingService.get();
-      cfg.GRPC_PROXY = GRPC_PROXY ? GRPC_PROXY : '';
+      cfg.GRPC_PROXY = GRPC_PROXY || '';
       await cfg.validate();
 
-      const dependencies = await new DependencyScanner(cfg).scan(allFiles);
+      const chunks = [];
+      for (let i = 0; i < allFiles.length; i += AppConfigModule.DEFAULT_SERVICE_CHUNK_LIMIT) {
+        chunks.push(allFiles.slice(i, i + 10));
+      }
+      const depScanner = new DependencyScanner(cfg);
+      const promises = chunks.map(async (chunk) => {
+        try {
+          return await depScanner.scan(chunk);
+        } catch (err: any) {
+          log.error('[ DependencyTask ] Request failed for purls:', chunk.map((file: any) => file));
+          log.error('Error:', err);
+          return null;
+        }
+      });
+      const results = await Promise.all(promises);
+
+      const dependencies = results.reduce((acc, curr) => {
+        if (!curr) return acc;
+        return {
+          filesList: [...(acc.filesList || []), ...(curr.filesList || [])],
+        };
+      }, { filesList: [] });
       dependencies.filesList.forEach((f) => {
         f.file = f.file.replace(rootPath, '');
       });

diff --git a/src/main/task/vulnerability/AddVulnerabilityTask.ts b/src/main/task/vulnerability/AddVulnerabilityTask.ts
@@ -10,6 +10,8 @@ import { gRPCConnections } from '../grpc/gRPCConnection/gRPCConnection';
 import { GetVulnerabilityRequestBuilder } from './builders/GetVulnerabilityRequestBuilder';
 import { VulnerabilitiesClient } from '../grpc/scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities_grpc_pb';
 
+
+
 export interface IVulnerabilityTask {
   components: Array<string>;
   force?:boolean;
@@ -26,7 +28,7 @@ export class AddVulnerabilityTask implements ITask<IVulnerabilityTask, void> {
       if (params.force) await modelProvider.model.vulnerability.deleteAll();
       await this.insertVulnerabilities(respJSON);
     } catch (err: any) {
-      log.error("[Add Vulnerability Task]:", err);
+      log.error('[Add Vulnerability Task]:', err);
       throw err;
     }
   }
@@ -41,24 +43,53 @@ export class AddVulnerabilityTask implements ITask<IVulnerabilityTask, void> {
 
   private async scanVulnerabilities(components: Array<string>) {
     const client = gRPCConnections.getVulnerabilityStub() as VulnerabilitiesClient;
-    const data = {
-      purls: components.map((item) => ({ purl: item })),
-    };
 
-    const req = GetVulnerabilityRequestBuilder.build(data);
+    // Split components into chunks
+    const chunks = [];
+    for (let i = 0; i < components.length; i += AppConfig.DEFAULT_SERVICE_CHUNK_LIMIT) {
+      chunks.push(components.slice(i, i + 10));
+    }
 
-    const pGetVulnerabilities = new Promise((resolve, reject) => {
-      client.getVulnerabilities(req, (err, resp) => {
-        if (err){
-          reject(err);
-        } 
-        else resolve(resp);
-      });
+    // Process each chunk and collect promises
+    const promises = chunks.map(async (chunk) => {
+      const data = {
+        purls: chunk.map((item) => ({ purl: item })),
+      };
+
+      const req = GetVulnerabilityRequestBuilder.build(data);
+      try {
+        const response = await new Promise((resolve, reject) => {
+          client.getVulnerabilities(req, (err, resp) => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve(resp);
+            }
+          });
+        }) as VulnerabilityMessages.VulnerabilityResponse;
+
+        if (response.getStatus().getStatus() !== StatusCode.SUCCESS) {
+          log.error('[ VulnerabilityTask ] Request failed for purls:', chunk.map((item: any) => item.purl));
+          log.error(`Error message: ${response.getStatus().getMessage()}`);
+          return null;
+        }
+        return response.toObject();
+      } catch (err: any) {
+        log.error('[ VulnerabilityTask ] Request failed for purls:', chunk.map((item: any) => item.purl));
+        log.error('Error:', err);
+        return null;
+      }
     });
 
-    const response = (await pGetVulnerabilities) as VulnerabilityMessages.VulnerabilityResponse;
-    if (response.getStatus().getStatus() !== StatusCode.SUCCESS) throw new Error(response.getStatus().getMessage());
-    return response.toObject();
+    const results = await Promise.all(promises);
+    return results.reduce((acc, curr) => {
+      if (!curr) return acc;
+
+      return {
+        purlsList: [...(acc.purlsList || []), ...(curr.purlsList || [])],
+        status: curr.status,
+      };
+    }, { purlsList: [], status: null });
   }
 
   private groupVulnerabilitiesByCVE(vulnerabilities: any): Array<Vulnerability> {