Lazy AllPackages

.changeset/slimy-pumpkins-camp.md

@@ -0,0 +1,9 @@
+---
+"@definitelytyped/definitions-parser": patch
+"@definitelytyped/dtslint-runner": patch
+"@definitelytyped/publisher": patch
+"@definitelytyped/retag": patch
+"@definitelytyped/utils": patch
+---
+
+Make AllPackages lazy and asynchronous

packages/definitions-parser/src/get-affected-packages.ts

@@ -31,7 +31,7 @@ export async function getAffectedPackages(
   const allDependentDirectories = [];
   const filters = [`--filter '...[${sourceRemote}/${sourceBranch}]'`];
   for (const d of deletions) {
-    for (const dep of allPackages.allTypings()) {
+    for (const dep of await allPackages.allTypings()) {
       for (const [name, version] of dep.allPackageJsonDependencies()) {
         if (
           "@types/" + d.typesDirectoryName === name &&
@@ -61,36 +61,40 @@ export async function getAffectedPackages(
   );
 }
 /** This function is exported for testing, since it's determined entirely by its inputs. */
-export function getAffectedPackagesWorker(
+export async function getAffectedPackagesWorker(
   allPackages: AllPackages,
   changedOutput: string,
   additions: string[],
   dependentOutputs: string[],
   definitelyTypedPath: string
-): PreparePackagesResult {
+): Promise<PreparePackagesResult> {
   const dt = resolve(definitelyTypedPath);
   const changedDirs = mapDefined(changedOutput.split("\n"), getDirectoryName(dt));
   const dependentDirs = mapDefined(dependentOutputs.join("\n").split("\n"), getDirectoryName(dt));
   const packageNames = new Set([
     ...additions,
-    ...changedDirs.map(
-      (c) =>
+    ...await Promise.all(changedDirs.map(
+      async (c) =>
         assertDefined(
-          allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(c + "/index.d.ts"), "bad path " + c)),
+          await allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(c + "/index.d.ts"), "bad path " + c)),
           "bad path " + JSON.stringify(getDependencyFromFile(c + "/index.d.ts"))
         ).subDirectoryPath
-    ),
+    )),
   ]);
   const dependents = new Set(
-    dependentDirs
-      .map(
-        (d) =>
-          assertDefined(
-            allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(d + "/index.d.ts"), "bad path " + d)),
-            d + " package not found"
-          ).subDirectoryPath
+    (
+      await Promise.all(
+        dependentDirs.map(
+          async (d) =>
+            assertDefined(
+              await allPackages.tryGetTypingsData(
+                assertDefined(getDependencyFromFile(d + "/index.d.ts"), "bad path " + d)
+              ),
+              d + " package not found"
+            ).subDirectoryPath
+        )
       )
-      .filter((d) => !packageNames.has(d))
+    ).filter((d) => !packageNames.has(d))
   );
   return { packageNames, dependents };
 }

packages/definitions-parser/src/get-definitely-typed.ts

@@ -15,8 +15,6 @@ export interface ParseDefinitionsOptions {
   readonly definitelyTypedPath: string | undefined;
   /** Whether to show progress bars. Good when running locally, bad when running in CI. */
   readonly progress: boolean;
-  /** Disabled in CI since it has problems logging errors from other processes. */
-  readonly parseInParallel: boolean;
 }
 
 export async function getDefinitelyTyped(options: ParseDefinitionsOptions, log: LoggerWithErrors): Promise<FS> {

packages/definitions-parser/src/git.ts

@@ -86,7 +86,7 @@ export async function getAffectedPackagesFromDiff(
   const errors = [];
   const diffs = await gitDiff(consoleLogger.info, definitelyTypedPath);
   if (diffs.find((d) => d.file === "notNeededPackages.json")) {
-    const deleteds = getNotNeededPackages(allPackages, diffs);
+    const deleteds = await getNotNeededPackages(allPackages, diffs);
     if ("errors" in deleteds) errors.push(...deleteds.errors);
     else
       for (const deleted of deleteds) {
@@ -139,18 +139,18 @@ it is supposed to replace, ${typings.version} of ${unneeded.name}.`);
  * 1. Find all the deleted files and group by package (error on deleted files outside a package).
  * 2. Make sure that all deleted packages in notNeededPackages have no files left.
  */
-export function getNotNeededPackages(
+export async function getNotNeededPackages(
   allPackages: AllPackages,
   diffs: GitDiff[]
-): { errors: string[] } | NotNeededPackage[] {
+): Promise<{ errors: string[] } | NotNeededPackage[]> {
   const changes = gitChanges(diffs);
   if ("errors" in changes) return changes;
   const { deletions } = changes;
   const deletedPackages = new Set(deletions.map((p) => assertDefined(p.typesDirectoryName)));
   const notNeededs = [];
   const errors = [];
   for (const p of deletedPackages) {
-    const hasTyping = allPackages.hasTypingFor({ typesDirectoryName: p, version: "*" });
+    const hasTyping = await allPackages.hasTypingFor({ typesDirectoryName: p, version: "*" });
     const notNeeded = allPackages.getNotNeededPackage(p);
     if (hasTyping && notNeeded) {
       errors.push(`Please delete all files in ${p} when adding it to notNeededPackages.json.`);

packages/definitions-parser/src/index.ts

@@ -3,6 +3,5 @@ export * from "./data-file";
 export * from "./get-affected-packages";
 export * from "./get-definitely-typed";
 export * from "./git";
-export * from "./parse-definitions";
 export * from "./mocks";
 export * from "./packages";