Merge pull request #379 from snyk-tech-services/feat/increse-project-…

…limit feat: bump project API limit from 10 to 100 for speed
snyk · Nov 8, 2022 · 9566af2 · 9566af2
2 parents 45c7146 + f40a66d
commit 9566af2
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 18 deletions.
diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts
@@ -42,7 +42,7 @@ export async function listIntegrations(
   if (!statusCode || statusCode !== 200) {
     throw new Error(
       'Expected a 200 response, instead received: ' +
-        JSON.stringify({ data: res.data || res.body, status: statusCode }),
+      JSON.stringify({ data: res.data || res.body, status: statusCode }),
     );
   }
   return res.data || {};
@@ -100,7 +100,7 @@ export async function setNotificationPreferences(
     if (!statusCode || statusCode !== 200) {
       throw new Error(
         'Expected a 200 response, instead received: ' +
-          JSON.stringify({ data: res.data, status: statusCode }),
+        JSON.stringify({ data: res.data, status: statusCode }),
       );
     }
     return res.data || {};
@@ -133,7 +133,7 @@ export async function deleteOrg(
   if (!statusCode || statusCode !== 204) {
     throw new Error(
       'Expected a 204 response, instead received: ' +
-        JSON.stringify({ data: res.data, status: statusCode }),
+      JSON.stringify({ data: res.data, status: statusCode }),
     );
   }
   return res.data;
@@ -166,7 +166,8 @@ interface ProjectsFilters {
   origin?: string; //If supplied, only projects that exactly match this origin will be returned
   type?: string; //If supplied, only projects that exactly match this type will be returned
   isMonitored?: boolean; // If set to true, only include projects which are monitored, if set to false, only include projects which are not monitored
-  targetId?: string; // The target ID to be used in sunc functions
+  targetId?: string; // The target public ID
+  limit?: number; // how many results per page, defaults to 10
 }
 
 export async function listProjects(
@@ -186,14 +187,12 @@ export async function listProjects(
 
   const projects = await listAllProjects(requestManager, orgId, filters);
 
-  const snykProjectData: ProjectsResponse = {
+  return {
     org: {
       id: orgId,
     },
     projects: projects,
   };
-
-  return snykProjectData;
 }
 
 async function listAllProjects(
@@ -216,7 +215,7 @@ async function listAllProjects(
       }: {
         projects: SnykProject[];
         next?: string;
-      } = await getProject(requestManager, orgId, filters, nextPageLink);
+      } = await getProjectsPage(requestManager, orgId, filters, nextPageLink);
 
       projectsList.push(...projects);
       next
@@ -231,7 +230,7 @@ async function listAllProjects(
   return projectsList;
 }
 
-async function getProject(
+async function getProjectsPage(
   requestManager: requestsManager,
   orgId: string,
   filters?: ProjectsFilters,
@@ -255,7 +254,7 @@ async function getProject(
   if (!statusCode || statusCode !== 200) {
     throw new Error(
       'Expected a 200 response, instead received: ' +
-        JSON.stringify({ data: res.data, status: statusCode }),
+      JSON.stringify({ data: res.data, status: statusCode }),
     );
   }
 
@@ -357,9 +356,9 @@ export async function getSnykTarget(
     excludeEmpty?: boolean;
     origin?: string;
   } = {
-    limit: 20,
-    excludeEmpty: true,
-  },
+      limit: 20,
+      excludeEmpty: true,
+    },
 ): Promise<{ targets: SnykTarget[]; next?: string }> {
   const query = qs.stringify({
     version: '2022-09-15~beta',
@@ -378,7 +377,7 @@ export async function getSnykTarget(
   if (!statusCode || statusCode !== 200) {
     throw new Error(
       'Expected a 200 response, instead received: ' +
-        JSON.stringify({ data: res.data, status: statusCode }),
+      JSON.stringify({ data: res.data, status: statusCode }),
     );
   }
 
@@ -388,3 +387,4 @@ export async function getSnykTarget(
 
   return { targets, next };
 }
+
diff --git a/src/scripts/generate-imported-targets-from-snyk.ts b/src/scripts/generate-imported-targets-from-snyk.ts
@@ -118,7 +118,7 @@ export async function generateSnykImportedTargets(
     ? await getAllOrgs(requestManager, groupId)
     : [{ id: orgId! }];
   const failedOrgs: SnykOrg[] = [];
-  const projectFilters = integrationTypes.length > 1 ?  undefined: { origin: integrationTypes[0] };
+  const projectFilters = integrationTypes.length > 1 ? { limit: 100 } : { origin: integrationTypes[0], limit: 100 };
   await pMap(
     groupOrgs,
     async (org: SnykOrg) => {
@@ -183,8 +183,7 @@ export async function generateSnykImportedTargets(
       } catch (e) {
         failedOrgs.push(org);
         console.warn(
-          `Failed to process projects for organization ${
-            name && slug ? `${name}(${slug})` : orgId
+          `Failed to process projects for organization ${name && slug ? `${name}(${slug})` : orgId
           }. Continuing.`,
         );
       }

diff --git a/src/scripts/sync/sync-org-projects.ts b/src/scripts/sync/sync-org-projects.ts
@@ -127,7 +127,7 @@ export async function updateTargets(
     targets,
     async (target: SnykTarget) => {
       try {
-        const filters = { targetId: target.id };
+        const filters = { targetId: target.id, limit: 100 };
         debug(`Listing projects for target ${target.attributes.displayName}`);
         const { projects } = await listProjects(requestManager, orgId, filters);
         debug(`Syncing projects for target ${target.attributes.displayName}`);