From 6593e998e803a0e6c68c815f00769c5c8aac5ed2 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 18 Oct 2023 14:04:36 -0700 Subject: [PATCH 01/12] Get it building --- .../src/get-affected-packages.ts | 40 ++-- packages/definitions-parser/src/index.ts | 1 - .../src/lib/definition-parser-worker.ts | 23 -- .../src/lib/definition-parser.ts | 163 +++++++-------- packages/definitions-parser/src/packages.ts | 196 +++++++++++------- .../src/parse-definitions.ts | 72 ------- .../test/get-affected-packages.test.ts | 22 +- packages/definitions-parser/test/git.test.ts | 10 +- .../definitions-parser/test/packages.test.ts | 47 +++-- .../test/parse-definitions.test.ts | 30 --- packages/definitions-parser/test/utils.ts | 2 - .../dtslint-runner/src/check-parse-results.ts | 10 +- .../src/prepareAffectedPackages.ts | 14 +- .../dtslint-runner/src/prepareAllPackages.ts | 8 +- packages/publisher/package.json | 1 - packages/publisher/src/calculate-versions.ts | 6 +- packages/publisher/src/full.ts | 14 +- packages/publisher/src/generate-packages.ts | 2 +- packages/publisher/src/lib/versions.ts | 6 +- packages/publisher/src/parse-definitions.ts | 46 ---- packages/publisher/src/publish-packages.ts | 2 +- packages/publisher/src/publish-registry.ts | 4 +- .../publisher/test/generate-packages.test.ts | 32 +-- packages/retag/src/index.ts | 15 +- 24 files changed, 315 insertions(+), 451 deletions(-) delete mode 100644 packages/definitions-parser/src/lib/definition-parser-worker.ts delete mode 100644 packages/definitions-parser/src/parse-definitions.ts delete mode 100644 packages/definitions-parser/test/parse-definitions.test.ts delete mode 100644 packages/publisher/src/parse-definitions.ts diff --git a/packages/definitions-parser/src/get-affected-packages.ts b/packages/definitions-parser/src/get-affected-packages.ts index 7f3c5ca325..d03dad7ccc 100644 --- a/packages/definitions-parser/src/get-affected-packages.ts +++ b/packages/definitions-parser/src/get-affected-packages.ts @@ -17,7 +17,7 @@ export async function getAffectedPackages( const allDependents = []; const filters = [`--filter '...[${sourceRemote}/${sourceBranch}]'`]; for (const d of deletions) { - for (const dep of allPackages.allTypings()) { + for (const dep of await allPackages.allTypings()) { for (const [name, version] of dep.allPackageJsonDependencies()) { if ( "@types/" + d.typesDirectoryName === name && @@ -45,33 +45,41 @@ export async function getAffectedPackages( return getAffectedPackagesWorker(allPackages, changedPackageNames, allDependents, definitelyTypedPath); } /** This function is exported for testing, since it's determined entirely by its inputs. */ -export function getAffectedPackagesWorker( +export async function getAffectedPackagesWorker( allPackages: AllPackages, changedOutput: string, dependentOutputs: string[], definitelyTypedPath: string -): PreparePackagesResult { +): Promise { const dt = resolve(definitelyTypedPath); const changedDirs = mapDefined(changedOutput.split("\n"), getDirectoryName(dt)); const dependentDirs = mapDefined(dependentOutputs.join("\n").split("\n"), getDirectoryName(dt)); const packageNames = new Set( - changedDirs.map( - (c) => - assertDefined( - allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(c + "/index.d.ts"), "bad path " + c)) - ).subDirectoryPath + await Promise.all( + changedDirs.map( + async (c) => + assertDefined( + await allPackages.tryGetTypingsData( + assertDefined(getDependencyFromFile(c + "/index.d.ts"), "bad path " + c) + ) + ).subDirectoryPath + ) ) ); const dependents = new Set( - dependentDirs - .map( - (d) => - assertDefined( - allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(d + "/index.d.ts"), "bad path " + d)), - d + " package not found" - ).subDirectoryPath + ( + await Promise.all( + dependentDirs.map( + async (d) => + assertDefined( + await allPackages.tryGetTypingsData( + assertDefined(getDependencyFromFile(d + "/index.d.ts"), "bad path " + d) + ), + d + " package not found" + ).subDirectoryPath + ) ) - .filter((d) => !packageNames.has(d)) + ).filter((d) => !packageNames.has(d)) ); return { packageNames, dependents }; } diff --git a/packages/definitions-parser/src/index.ts b/packages/definitions-parser/src/index.ts index 21350bd7f6..1589a9de6a 100644 --- a/packages/definitions-parser/src/index.ts +++ b/packages/definitions-parser/src/index.ts @@ -3,6 +3,5 @@ export * from "./data-file"; export * from "./get-affected-packages"; export * from "./get-definitely-typed"; export * from "./git"; -export * from "./parse-definitions"; export * from "./mocks"; export * from "./packages"; diff --git a/packages/definitions-parser/src/lib/definition-parser-worker.ts b/packages/definitions-parser/src/lib/definition-parser-worker.ts deleted file mode 100644 index f1d2df98ff..0000000000 --- a/packages/definitions-parser/src/lib/definition-parser-worker.ts +++ /dev/null @@ -1,23 +0,0 @@ -import assert = require("assert"); -import process = require("process"); -import { logUncaughtErrors } from "@definitelytyped/utils"; -import { getLocallyInstalledDefinitelyTyped } from "../get-definitely-typed"; -import { getTypingInfo } from "./definition-parser"; -import { dirname } from "path"; - -// This file is "called" by runWithChildProcesses from parse-definition.ts -export const definitionParserWorkerFilename = __filename; - -if (require.main === module) { - process.on("message", (message) => { - assert(process.argv.length === 3); - const typesPath = process.argv[2]; - // tslint:disable-next-line no-async-without-await - logUncaughtErrors(async () => { - for (const packageName of message as readonly string[]) { - const data = await getTypingInfo(packageName, getLocallyInstalledDefinitelyTyped(dirname(typesPath))); - process.send!({ data, packageName }); - } - }); - }); -} diff --git a/packages/definitions-parser/src/lib/definition-parser.ts b/packages/definitions-parser/src/lib/definition-parser.ts index 874ce826c8..6897e261f8 100644 --- a/packages/definitions-parser/src/lib/definition-parser.ts +++ b/packages/definitions-parser/src/lib/definition-parser.ts @@ -1,32 +1,29 @@ import { - validatePackageJson, License, - getLicenseFromPackageJson, - checkPackageJsonType, checkPackageJsonDependencies, - checkPackageJsonImports, checkPackageJsonExportsAndAddPJsonEntry, + checkPackageJsonImports, + checkPackageJsonType, + getLicenseFromPackageJson, + validatePackageJson, } from "@definitelytyped/header-parser"; import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; import { FS, assertDefined, - computeHash, - createModuleResolutionHost, filter, - flatMap, hasWindowsSlashes, join, - mapDefined, sort, split, - withoutStart, + withoutStart } from "@definitelytyped/utils"; import assert from "assert"; import path from "path"; import * as ts from "typescript"; import { DirectoryParsedTypingVersion, + TypingsData, TypingsDataRaw, TypingsVersionsRaw, formatTypingVersion, @@ -53,17 +50,17 @@ function formattedLibraryVersion(typingsDataRaw: TypingsDataRaw): `${number}.${n return `${typingsDataRaw.header.libraryMajorVersion}.${typingsDataRaw.header.libraryMinorVersion}`; } -export async function getTypingInfo(packageName: string, dt: FS): Promise { +export async function getTypingInfo(packageNameOrTypesDirectoryName: string, dt: FS): Promise { const errors = []; - if (packageName !== packageName.toLowerCase()) { - errors.push(`Package name \`${packageName}\` should be strictly lowercase`); + if (packageNameOrTypesDirectoryName !== packageNameOrTypesDirectoryName.toLowerCase()) { + errors.push(`Package name \`${packageNameOrTypesDirectoryName}\` should be strictly lowercase`); } interface OlderVersionDir { readonly directoryName: string; readonly version: DirectoryParsedTypingVersion; } - const fs = dt.subDir("types").subDir(getMangledNameForScopedPackage(packageName)); + const fs = dt.subDir("types").subDir(getMangledNameForScopedPackage(packageNameOrTypesDirectoryName)); const [rootDirectoryLs, olderVersionDirectories] = split( fs.readdir(), (fileOrDirectoryName) => { @@ -72,9 +69,8 @@ export async function getTypingInfo(packageName: string, dt: FS): Promise version.minor !== undefined); - const latestDataResult = await combineDataForAllTypesVersions(packageName, rootDirectoryLs, fs, moduleResolutionHost); + const latestDataResult = await getPackageJsonInfoForPackage(packageNameOrTypesDirectoryName, rootDirectoryLs, fs); if (Array.isArray(latestDataResult)) { return { errors: [...errors, ...latestDataResult] }; } @@ -85,7 +81,7 @@ export async function getTypingInfo(packageName: string, dt: FS): Promise | string[]> { const errors = []; const typesVersionAndPackageJson = getTypesVersionsAndPackageJson(ls); if (Array.isArray(typesVersionAndPackageJson)) { errors.push(...typesVersionAndPackageJson); } - const { remainingLs, typesVersions } = Array.isArray(typesVersionAndPackageJson) - ? { remainingLs: [], typesVersions: [] } + const { typesVersions } = Array.isArray(typesVersionAndPackageJson) + ? { typesVersions: [] } : typesVersionAndPackageJson; const packageJson = fs.readJson(packageJsonName) as { readonly license?: unknown; @@ -209,30 +203,7 @@ async function combineDataForAllTypesVersions( readonly exports?: unknown; readonly type?: unknown; }; - const dataForRoot = getTypingDataForSingleTypesVersion( - undefined, - typingsPackageName, - remainingLs, - fs, - moduleResolutionHost - ); - if (Array.isArray(dataForRoot)) { - errors.push(...dataForRoot); - } - const dataForOtherTypesVersions = typesVersions.map((tsVersion) => { - const subFs = fs.subDir(`ts${tsVersion}`); - const data = getTypingDataForSingleTypesVersion( - tsVersion, - typingsPackageName, - subFs.readdir(), - subFs, - moduleResolutionHost - ); - if (Array.isArray(data)) { - errors.push(...data); - } - return data; - }); + const packageJsonType = checkPackageJsonType(packageJson.type, packageJsonName); if (Array.isArray(packageJsonType)) { errors.push(...packageJsonType); @@ -269,33 +240,71 @@ async function combineDataForAllTypesVersions( return errors; } - const allTypesVersions = [dataForRoot, ...dataForOtherTypesVersions] as TypingDataFromIndividualTypeScriptVersion[]; - const files = Array.from( - flatMap(allTypesVersions, ({ typescriptVersion, declFiles }) => - declFiles.map((file) => (typescriptVersion === undefined ? file : `ts${typescriptVersion}/${file}`)) - ) - ); - // Note that only the first project is collected right now return { header: assertDefined(header), typesVersions, - files, license: license as License, dependencies: packageJson.dependencies as Record, devDependencies: packageJson.devDependencies as Record, - contentHash: hash( - [...files, packageJsonName], - mapDefined(allTypesVersions, (a) => a.tsconfigPathsForHash), - fs - ), imports: imports as object | undefined, exports: exports as string | object | undefined, type: packageJsonType as "module" | undefined, }; } -interface TypingDataFromIndividualTypeScriptVersion { +export function getFiles( + dt: FS, + typingsData: TypingsData, + moduleResolutionHost: ts.ModuleResolutionHost +): readonly FilesForSingleTypeScriptVersion[] | { errors: string[] } { + const errors = []; + const rootDir = dt.subDir("types").subDir(typingsData.typesDirectoryName); + const typesVersionAndPackageJson = getTypesVersionsAndPackageJson(rootDir.readdir()); + if (Array.isArray(typesVersionAndPackageJson)) { + errors.push(...typesVersionAndPackageJson); + } + const { remainingLs, typesVersions } = Array.isArray(typesVersionAndPackageJson) + ? { remainingLs: [], typesVersions: [] } + : typesVersionAndPackageJson; + const dataForRoot = getFilesForSingleTypeScriptVersion( + undefined, + typingsData.typesDirectoryName, + remainingLs, + rootDir, + moduleResolutionHost + ); + if (Array.isArray(dataForRoot)) { + errors.push(...dataForRoot); + } + const dataForOtherTypesVersions = typesVersions.map((tsVersion) => { + const subFs = rootDir.subDir(`ts${tsVersion}`); + const data = getFilesForSingleTypeScriptVersion( + tsVersion, + typingsData.typesDirectoryName, + subFs.readdir(), + subFs, + moduleResolutionHost + ); + if (Array.isArray(data)) { + errors.push(...data); + } + return data; + }); + + if (errors.length) { + return { errors }; + } + + return [dataForRoot, ...dataForOtherTypesVersions] as FilesForSingleTypeScriptVersion[]; + // return Array.from( + // flatMap(allTypesVersions, ({ typescriptVersion, declFiles }) => + // declFiles.map((file) => (typescriptVersion === undefined ? file : `ts${typescriptVersion}/${file}`)) + // ) + // ); +} + +export interface FilesForSingleTypeScriptVersion { /** Undefined for root (which uses typeScriptVersion in package.json instead) */ readonly typescriptVersion: TypeScriptVersion | undefined; readonly declFiles: readonly string[]; // TODO: Used to map file.d.ts to ts4.1/file.d.ts -- not sure why this is needed @@ -304,17 +313,17 @@ interface TypingDataFromIndividualTypeScriptVersion { /** * @param typescriptVersion Set if this is in e.g. a `ts3.1` directory. - * @param packageName Name of the outermost directory; e.g. for "node/v4" this is just "node". + * @param typesDirectoryName Name of the outermost directory; e.g. for "node/v4" this is just "node". * @param ls All file/directory names in `directory`. * @param fs FS rooted at the directory for this particular TS version, e.g. `types/abs/ts3.1` or `types/abs` when typescriptVersion is undefined. */ -function getTypingDataForSingleTypesVersion( +function getFilesForSingleTypeScriptVersion( typescriptVersion: TypeScriptVersion | undefined, - packageName: string, + typesDirectoryName: string, ls: readonly string[], fs: FS, moduleResolutionHost: ts.ModuleResolutionHost -): TypingDataFromIndividualTypeScriptVersion | string[] { +): FilesForSingleTypeScriptVersion | string[] { const errors = []; const tsconfig = fs.readJson("tsconfig.json") as TsConfig; const configHost: ts.ParseConfigHost = { @@ -328,17 +337,17 @@ function getTypingDataForSingleTypesVersion( configHost, path.resolve("/", fs.debugPath()) ).options; - errors.push(...checkFilesFromTsConfig(packageName, tsconfig, fs.debugPath())); + errors.push(...checkFilesFromTsConfig(typesDirectoryName, tsconfig, fs.debugPath())); const { types, tests } = allReferencedFiles( tsconfig.files ?? [], fs, - packageName, + typesDirectoryName, moduleResolutionHost, compilerOptions ); const usedFiles = new Set( [...types.keys(), ...tests, "tsconfig.json", "tslint.json"].map((f) => - slicePrefixes(f, "node_modules/@types/" + packageName + "/") + slicePrefixes(f, "node_modules/@types/" + typesDirectoryName + "/") ) ); const otherFiles = ls.includes(unusedFilesName) @@ -349,15 +358,15 @@ function getTypingDataForSingleTypesVersion( .filter(Boolean) : []; if (ls.includes(unusedFilesName) && !otherFiles.length) { - errors.push(`In ${packageName}: OTHER_FILES.txt is empty.`); + errors.push(`In ${typesDirectoryName}: OTHER_FILES.txt is empty.`); } for (const file of otherFiles) { if (!isRelativePath(file)) { - errors.push(`In ${packageName}: A path segment is empty or all dots ${file}`); + errors.push(`In ${typesDirectoryName}: A path segment is empty or all dots ${file}`); } } // Note: findAllUnusedFiles also modifies usedFiles and otherFiles and errors - const unusedFiles = findAllUnusedFiles(ls, usedFiles, otherFiles, errors, packageName, fs); + const unusedFiles = findAllUnusedFiles(ls, usedFiles, otherFiles, errors, typesDirectoryName, fs); if (unusedFiles.length) { errors.push( "\n\t* " + @@ -378,9 +387,10 @@ function getTypingDataForSingleTypesVersion( } if (errors.length) return errors; + const declFiles = sort(types.keys()); return { typescriptVersion, - declFiles: sort(types.keys()), + declFiles: typescriptVersion === undefined ? declFiles : declFiles.map(f => `ts${typescriptVersion}/${f}`), tsconfigPathsForHash: JSON.stringify(tsconfig.compilerOptions.paths), }; } @@ -434,15 +444,6 @@ interface TsConfig { compilerOptions: ts.CompilerOptions; } -function hash(files: readonly string[], tsconfigPathsForHash: readonly string[], fs: FS): string { - const fileContents = files.map((f) => `${f}**${readFileAndThrowOnBOM(f, fs)}`); - let allContent = fileContents.join("||"); - for (const path of tsconfigPathsForHash) { - allContent += path; - } - return computeHash(allContent); -} - export function readFileAndThrowOnBOM(fileName: string, fs: FS): string { // tslint:disable-next-line:non-literal-fs-path -- Not a reference to the fs package const text = fs.readFile(fileName); diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index 948c1f3a21..a05ed6258f 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -1,50 +1,48 @@ import assert = require("assert"); -import { Owner, Header, License } from "@definitelytyped/header-parser"; +import { Header, License, Owner } from "@definitelytyped/header-parser"; import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; -import { FS, assertDefined, assertSorted, mapValues, unique, unmangleScopedPackage } from "@definitelytyped/utils"; +import { Dir, FS, InMemoryFS, assertDefined, computeHash, createModuleResolutionHost, mapDefined, unique, unmangleScopedPackage } from "@definitelytyped/utils"; import * as semver from "semver"; -import { readDataFile } from "./data-file"; -import { parseVersionFromDirectoryName } from "./lib/definition-parser"; +import { FilesForSingleTypeScriptVersion, getFiles, getTypingInfo, parseVersionFromDirectoryName, readFileAndThrowOnBOM } from "./lib/definition-parser"; import { scopeName, typesDirectoryName } from "./lib/settings"; import { slicePrefixes } from "./lib/utils"; export class AllPackages { - static async read(dt: FS): Promise { - return AllPackages.from(await readTypesDataFile(), readNotNeededPackages(dt)); + static fromFS(dt: FS) { + return new AllPackages(dt, new Map(), readNotNeededPackages(dt)); } - static from(data: TypesDataFile, notNeeded: readonly NotNeededPackage[]): AllPackages { + static fromTestData(typingsVersionsRaw: Record, notNeeded: readonly NotNeededPackage[]) { + const fs = new InMemoryFS(new Dir(/*parent*/ undefined), ""); return new AllPackages( - mapValues(new Map(Object.entries(data)), (raw) => new TypingsVersions(raw)), - notNeeded - ); - } - - static async readTypings(): Promise { - return AllPackages.from(await readTypesDataFile(), []).allTypings(); - } - static async readLatestTypings(): Promise { - return AllPackages.from(await readTypesDataFile(), []).allLatestTypings(); + fs, + new Map(Object.entries(typingsVersionsRaw).map(([name, raw]) => [name, new TypingsVersions(fs, raw)])), + notNeeded, + ) } /** Use for `--single` tasks only. Do *not* call this in a loop! */ - static async readSingle(name: string): Promise { - const data = await readTypesDataFile(); - const raw = data[name]; - if (!raw) { + static async readSingle(dt: FS, name: string): Promise { + const data = await getTypingInfo(name, dt); + if ("errors" in data) { throw new Error(`Can't find package ${name}`); } - const versions = Object.values(raw); + const versions = Object.values(data); if (versions.length > 1) { throw new Error(`Package ${name} has multiple versions.`); } - return new TypingsData(versions[0], /*isLatest*/ true); + return new TypingsData(dt, versions[0], /*isLatest*/ true); } + private readonly errors: Map = new Map(); + private isComplete = false; + private moduleResolutionHost = createModuleResolutionHost(this.dt, this.dt.debugPath()) + private constructor( + private dt: FS, /** Keys are `typesDirectoryName` strings */ - private readonly data: ReadonlyMap, - private readonly notNeeded: readonly NotNeededPackage[] + private readonly types: Map, + private readonly notNeeded: readonly NotNeededPackage[], ) {} getNotNeededPackage(typesDirectoryName: string): NotNeededPackage | undefined { @@ -54,27 +52,31 @@ export class AllPackages { hasTypingFor(dep: PackageId): boolean { return this.tryGetTypingsData(dep) !== undefined; } + + getErrors() { + return Array.from(this.errors.entries()).map(([name, errors]) => `${name}: ${errors.join("\n")}`); + } /** * Whether a package maintains multiple minor versions of typings simultaneously by * using minor-versioned directories like 'react-native/v14.1' */ - hasSeparateMinorVersions(typesDirectoryName: string) { - const versions = Array.from(assertDefined(this.data.get(typesDirectoryName)).getAll()); + async hasSeparateMinorVersions(typesDirectoryName: string) { + const versions = Array.from(assertDefined(await this.tryGetTypingsVersions(typesDirectoryName)).getAll()); const minors = versions.map((v) => v.minor); return minors.length !== unique(minors).length; } - tryResolve(dep: PackageId): PackageId { + async tryResolve(dep: PackageId): Promise { const typesDirectoryName = dep.typesDirectoryName ?? dep.name.slice(scopeName.length + 2); - const versions = this.data.get(typesDirectoryName); + const versions = await this.tryGetTypingsVersions(typesDirectoryName); const depVersion = new semver.Range(dep.version === "*" ? "*" : `^${formatTypingVersion(dep.version)}`); return (versions && versions.tryGet(depVersion)?.id) || dep; } - resolve(dep: PackageId): PackageIdWithDefiniteVersion { + async resolve(dep: PackageId): Promise { const typesDirectoryName = dep.typesDirectoryName ?? dep.name.slice(scopeName.length + 2); - const versions = this.data.get(typesDirectoryName); + const versions = await this.tryGetTypingsVersions(typesDirectoryName); if (!versions) { throw new Error(`No typings found with directory name '${dep.typesDirectoryName}'.`); } @@ -83,53 +85,71 @@ export class AllPackages { } /** Gets the latest version of a package. E.g. getLatest(node v6) was node v10 (before node v11 came out). */ - getLatest(pkg: TypingsData): TypingsData { + async getLatest(pkg: TypingsData): Promise { return pkg.isLatest ? pkg : this.getLatestVersion(pkg.typesDirectoryName); } - private getLatestVersion(typesDirectoryName: string): TypingsData { - const latest = this.tryGetLatestVersion(typesDirectoryName); + private async getLatestVersion(typesDirectoryName: string): Promise { + const latest = await this.tryGetLatestVersion(typesDirectoryName); if (!latest) { throw new Error(`No such package ${typesDirectoryName}.`); } return latest; } - tryGetLatestVersion(typesDirectoryName: string): TypingsData | undefined { - const versions = this.data.get(typesDirectoryName); + async tryGetLatestVersion(typesDirectoryName: string): Promise { + const versions = await this.tryGetTypingsVersions(typesDirectoryName); return versions && versions.getLatest(); } - getTypingsData(id: PackageId): TypingsData { - const pkg = this.tryGetTypingsData(id); + async getTypingsData(id: PackageId): Promise { + const pkg = await this.tryGetTypingsData(id); if (!pkg) { throw new Error(`No typings available for ${JSON.stringify(id)}`); } return pkg; } - tryGetTypingsData(pkg: PackageId): TypingsData | undefined { + async tryGetTypingsData(pkg: PackageId): Promise { const typesDirectoryName = pkg.typesDirectoryName ?? pkg.name.slice(scopeName.length + 2); - const versions = this.data.get(typesDirectoryName); + const versions = await this.tryGetTypingsVersions(typesDirectoryName); return ( versions && versions.tryGet(new semver.Range(pkg.version === "*" ? "*" : `^${formatTypingVersion(pkg.version)}`)) ); } + + async tryGetTypingsVersions(typesDirectoryName: string): Promise { + let versions = this.types.get(typesDirectoryName); + if (versions) { + return versions; + } + if (this.errors.has(typesDirectoryName)) { + return undefined; + } + const raw = await getTypingInfo(typesDirectoryName, this.dt); + if ("errors" in raw) { + this.errors.set(typesDirectoryName, raw.errors); + return undefined; + } + versions = new TypingsVersions(this.dt, raw, this.moduleResolutionHost); + this.types.set(typesDirectoryName, versions); + return versions; + } + - allPackages(): readonly AnyPackage[] { - return [...this.allTypings(), ...this.allNotNeeded()]; + async allPackages(): Promise { + return [...await this.allTypings(), ...this.allNotNeeded()]; } /** Note: this includes older version directories (`foo/v0`) */ - allTypings(): readonly TypingsData[] { - return assertSorted(Array.from(flattenData(this.data)), (t) => t.name); + async allTypings(): Promise { + await this.readAllTypings(); + return Array.from(flattenData(this.types)); } - allLatestTypings(): readonly TypingsData[] { - return assertSorted( - Array.from(this.data.values()).map((versions) => versions.getLatest()), - (t) => t.name - ); + async allLatestTypings(): Promise { + await this.readAllTypings(); + return Array.from(this.types.values()).map((versions) => versions.getLatest()); } allNotNeeded(): readonly NotNeededPackage[] { @@ -137,17 +157,31 @@ export class AllPackages { } /** Returns all of the dependencies *that are typed on DT*, ignoring others, and including test dependencies. */ - *allDependencyTypings(pkg: TypingsData): Iterable { + async *allDependencyTypings(pkg: TypingsData): AsyncIterable { for (const [name, version] of pkg.allPackageJsonDependencies()) { if (!name.startsWith(`@${scopeName}/`)) continue; if (pkg.name === name) continue; const typesDirectoryName = removeTypesScope(name); - const versions = this.data.get(typesDirectoryName); + const versions = await this.tryGetTypingsVersions(typesDirectoryName); if (versions) { yield versions.get(new semver.Range(version), pkg.name + ":" + JSON.stringify((versions as any).versions)); } } } + + private async readAllTypings() { + if (this.isComplete) { + return; + } + const types = this.dt.subDir("types"); + for (const typesDirectoryName of types.readdir()) { + if (!types.isDirectory(typesDirectoryName) || !types.subDir(typesDirectoryName).exists("package.json")) { + continue; + } + this.tryGetTypingsVersions(typesDirectoryName); + } + this.isComplete = true; + } } export function removeTypesScope(name: string) { @@ -165,8 +199,6 @@ export function getMangledNameForScopedPackage(packageName: string): string { return packageName; } -export const typesDataFilename = "definitions.json"; - function* flattenData(data: ReadonlyMap): Iterable { for (const versions of data.values()) { yield* versions.getAll(); @@ -347,24 +379,12 @@ export interface TypingsDataRaw { */ readonly typesVersions: readonly TypeScriptVersion[]; - /** - * Files that should be published with this definition, e.g. ["jquery.d.ts", "jquery-extras.d.ts"] - * - * Does *not* include `package.json` because that is not copied directly, but generated from TypingsData. - */ - readonly files: readonly string[]; - /** * The license that this definition package is released under. * * Can be either MIT or Apache v2, defaults to MIT when not explicitly defined in this package’s "package.json". */ readonly license: License; - - /** - * A hash of the names and contents of the `files` list, used for versioning. - */ - readonly contentHash: string; } export class TypingsVersions { @@ -375,7 +395,7 @@ export class TypingsVersions { */ private readonly versions: semver.SemVer[]; - constructor(data: TypingsVersionsRaw) { + constructor(dt: FS, data: TypingsVersionsRaw, private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath())) { /** * Sorted from latest to oldest so that we publish the current version first. * This is important because older versions repeatedly reset the "latest" tag to the current version. @@ -384,7 +404,7 @@ export class TypingsVersions { this.versions.sort(semver.rcompare); this.map = new Map( - this.versions.map((version, i) => [version, new TypingsData(data[`${version.major}.${version.minor}`], !i)]) + this.versions.map((version, i) => [version, new TypingsData(dt, data[`${version.major}.${version.minor}`], !i, this.moduleResolutionHost)]) ); } @@ -414,7 +434,9 @@ export class TypingsVersions { } export class TypingsData extends PackageBase { - constructor(private readonly data: TypingsDataRaw, readonly isLatest: boolean) { + public errors: readonly string[] | undefined; + + constructor(private dt: FS, private readonly data: TypingsDataRaw, readonly isLatest: boolean, private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath())) { super(); } @@ -445,11 +467,21 @@ export class TypingsData extends PackageBase { return this.data.typesVersions; } + private typesVersionsFiles: readonly FilesForSingleTypeScriptVersion[] | undefined; get files(): readonly string[] { - return this.data.files; + if (!this.typesVersionsFiles) { + const files = getFiles(this.dt, this, this.moduleResolutionHost); + if ("errors" in files) { + this.errors = files.errors; + this.typesVersionsFiles = []; + } else { + this.typesVersionsFiles = files; + } + } + return this.typesVersionsFiles.flatMap((v) => v.declFiles); } get dtsFiles(): readonly string[] { - return this.data.files.filter((f) => f.endsWith(".d.ts") || f.endsWith(".d.mts") || f.endsWith(".d.cts")); + return this.files.filter((f) => f.endsWith(".d.ts") || f.endsWith(".d.mts") || f.endsWith(".d.cts")); } get license(): License { return this.data.license; @@ -468,8 +500,14 @@ export class TypingsData extends PackageBase { yield [name, version]; } } + + private _contentHash: string | undefined; get contentHash(): string { - return this.data.contentHash; + return this._contentHash ??= hash( + [...this.files, "package.json"], + mapDefined(this.typesVersionsFiles!, (a) => a.tsconfigPathsForHash), + this.dt + ); } get projectName(): string | undefined { return this.data.header.projects[0]; @@ -517,13 +555,6 @@ export interface PackageIdWithDefiniteVersion { readonly version: HeaderParsedTypingVersion; } -export interface TypesDataFile { - readonly [packageName: string]: TypingsVersionsRaw; -} -function readTypesDataFile(): Promise { - return readDataFile("parse-definitions", typesDataFilename) as Promise; -} - export function readNotNeededPackages(dt: FS): readonly NotNeededPackage[] { const rawJson = dt.readJson("notNeededPackages.json"); // tslint:disable-line await-promise (tslint bug) return Object.entries((rawJson as { readonly packages: readonly NotNeededPackageRaw[] }).packages).map((entry) => @@ -560,3 +591,12 @@ export function getDependencyFromFile( return { typesDirectoryName: name, version: "*" }; } + +function hash(files: readonly string[], tsconfigPathsForHash: readonly string[], fs: FS): string { + const fileContents = files.map((f) => `${f}**${readFileAndThrowOnBOM(f, fs)}`); + let allContent = fileContents.join("||"); + for (const path of tsconfigPathsForHash) { + allContent += path; + } + return computeHash(allContent); +} \ No newline at end of file diff --git a/packages/definitions-parser/src/parse-definitions.ts b/packages/definitions-parser/src/parse-definitions.ts deleted file mode 100644 index 3d2122ed0a..0000000000 --- a/packages/definitions-parser/src/parse-definitions.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { FS, LoggerWithErrors, filterNAtATimeOrdered, joinPaths, runWithChildProcesses } from "@definitelytyped/utils"; -import { writeDataFile } from "./data-file"; -import { getTypingInfo } from "./lib/definition-parser"; -import { definitionParserWorkerFilename } from "./lib/definition-parser-worker"; -import { AllPackages, readNotNeededPackages, typesDataFilename, TypingsVersionsRaw } from "./packages"; - -export interface ParallelOptions { - readonly nProcesses: number; - readonly definitelyTypedPath: string; -} - -export async function parseDefinitions( - dt: FS, - parallel: ParallelOptions | undefined, - log: LoggerWithErrors -): Promise { - log.info("Parsing definitions..."); - const typesFS = dt.subDir("types"); - const packageNames = await filterNAtATimeOrdered( - parallel ? parallel.nProcesses : 1, - typesFS.readdir(), - (name) => typesFS.isDirectory(name) && typesFS.exists(joinPaths(name, "package.json")) - ); - log.info(`Found ${packageNames.length} packages.`); - - const typings: { [name: string]: TypingsVersionsRaw } = {}; - const errors: string[] = []; - - const start = Date.now(); - if (parallel) { - log.info(`Parsing in ${parallel.nProcesses} child process(es)...`); - await runWithChildProcesses({ - inputs: packageNames, - commandLineArgs: [`${parallel.definitelyTypedPath}/types`], - workerFile: definitionParserWorkerFilename, - nProcesses: parallel.nProcesses, - handleOutput({ data, packageName }: { data: TypingsVersionsRaw | { errors: string[] }; packageName: string }) { - if ("errors" in data) { - errors.push(...data.errors); - } else { - typings[packageName] = data; - } - }, - }); - } else { - const errors = []; - log.info("Parsing in main process..."); - for (const packageName of packageNames) { - const info = await getTypingInfo(packageName, dt); - if ("errors" in info) { - errors.push(...info.errors); - } else { - typings[packageName] = info; - } - } - } - if (errors.length > 0) { - throw new Error(errors.join("\n")); - } - log.info("Parsing took " + (Date.now() - start) / 1000 + " s"); - const sortedTypings = sorted(typings); - await writeDataFile(typesDataFilename, sortedTypings); - return AllPackages.from(sortedTypings, readNotNeededPackages(dt)); -} - -function sorted(obj: { [name: string]: T }): { [name: string]: T } { - const out: { [name: string]: T } = {}; - for (const key of Object.keys(obj).sort()) { - out[key] = obj[key]; - } - return out; -} diff --git a/packages/definitions-parser/test/get-affected-packages.test.ts b/packages/definitions-parser/test/get-affected-packages.test.ts index d104b3ae8a..007f728d87 100644 --- a/packages/definitions-parser/test/get-affected-packages.test.ts +++ b/packages/definitions-parser/test/get-affected-packages.test.ts @@ -1,8 +1,8 @@ import { getAffectedPackagesWorker } from "../src/get-affected-packages"; -import { NotNeededPackage, TypesDataFile, AllPackages } from "../src/packages"; +import { NotNeededPackage, AllPackages } from "../src/packages"; import { testo, createTypingsVersionRaw } from "./utils"; -const typesData: TypesDataFile = { +const typesData = { "has-older-test-dependency": createTypingsVersionRaw("has-older-test-dependency", {}, { "@types/jquery": "1.0.0" }), jquery: createTypingsVersionRaw("jquery", {}, {}), known: createTypingsVersionRaw("known", { "@types/jquery": "1.0.0" }, {}), @@ -17,15 +17,15 @@ typesData.jquery["2.0"] = { }; const notNeeded = [new NotNeededPackage("jest", "jest", "100.0.0")]; -const allPackages = AllPackages.from(typesData, notNeeded); +const allPackages = AllPackages.fromTestData(typesData, notNeeded); testo({ - updatedPackage() { + async updatedPackage() { const packageOutput = `/dt/types/jquery`; const dependentOutput = `/dt/types/jquery /dt/types/known-test /dt/types/most-recent`; - const { packageNames, dependents } = getAffectedPackagesWorker( + const { packageNames, dependents } = await getAffectedPackagesWorker( allPackages, packageOutput, [dependentOutput], @@ -34,10 +34,10 @@ testo({ expect(packageNames).toEqual(new Set(["jquery"])); expect(dependents).toEqual(new Set(["known-test", "most-recent"])); }, - deletedPackage() { + async deletedPackage() { const packageOutput = ``; const dependentOutput = `/dt/types/unknown-test`; - const { packageNames, dependents } = getAffectedPackagesWorker( + const { packageNames, dependents } = await getAffectedPackagesWorker( allPackages, packageOutput, [dependentOutput], @@ -46,7 +46,7 @@ testo({ expect(packageNames).toEqual(new Set([])); expect(dependents).toEqual(new Set(["unknown-test"])); }, - deletedVersion() { + async deletedVersion() { const packageOutput = `/dt/types/jquery`; const dependentOutput = [ `/dt/types/jquery @@ -55,15 +55,15 @@ testo({ `/dt/types/has-older-test-dependency /dt/types/known`, ]; - const { packageNames } = getAffectedPackagesWorker(allPackages, packageOutput, dependentOutput, "/dt"); + const { packageNames } = await getAffectedPackagesWorker(allPackages, packageOutput, dependentOutput, "/dt"); expect(packageNames).toEqual(new Set(["jquery"])); }, - olderVersion() { + async olderVersion() { const packageOutput = `/dt/types/jquery`; const dependentOutput = `/dt/types/jquery /dt/types/has-older-test-dependency /dt/types/known`; - const { packageNames, dependents } = getAffectedPackagesWorker( + const { packageNames, dependents } = await getAffectedPackagesWorker( allPackages, packageOutput, [dependentOutput], diff --git a/packages/definitions-parser/test/git.test.ts b/packages/definitions-parser/test/git.test.ts index 1584291c4f..872cd80039 100644 --- a/packages/definitions-parser/test/git.test.ts +++ b/packages/definitions-parser/test/git.test.ts @@ -2,9 +2,9 @@ import * as util from "util"; import * as pacote from "pacote"; import { createTypingsVersionRaw, testo } from "./utils"; import { GitDiff, getNotNeededPackages, checkNotNeededPackage } from "../src/git"; -import { NotNeededPackage, TypesDataFile, AllPackages } from "../src/packages"; +import { NotNeededPackage, AllPackages } from "../src/packages"; -const typesData: TypesDataFile = { +const typesData = { jquery: createTypingsVersionRaw("jquery", {}, {}), known: createTypingsVersionRaw("known", { "@types/jquery": "1.0.0" }, {}), "known-test": createTypingsVersionRaw("known-test", {}, { "@types/jquery": "*" }), @@ -14,7 +14,7 @@ const typesData: TypesDataFile = { }; const jestNotNeeded = [new NotNeededPackage("jest", "jest", "100.0.0")]; -const allPackages = AllPackages.from(typesData, jestNotNeeded); +const allPackages = AllPackages.fromTestData(typesData, jestNotNeeded); const deleteJestDiffs: GitDiff[] = [ { status: "M", file: "notNeededPackages.json" }, @@ -29,7 +29,7 @@ testo({ forgotToDeleteFiles() { expect( getNotNeededPackages( - AllPackages.from({ jest: createTypingsVersionRaw("jest", {}, {}) }, jestNotNeeded), + AllPackages.fromTestData({ jest: createTypingsVersionRaw("jest", {}, {}) }, jestNotNeeded), deleteJestDiffs ) ).toEqual({ errors: ["Please delete all files in jest when adding it to notNeededPackages.json."] }); @@ -60,7 +60,7 @@ When removing packages, you should only delete files that are a part of removed scoped() { expect( getNotNeededPackages( - AllPackages.from(typesData, [new NotNeededPackage("ember__object", "@ember/object", "1.0.0")]), + AllPackages.fromTestData(typesData, [new NotNeededPackage("ember__object", "@ember/object", "1.0.0")]), [{ status: "D", file: "types/ember__object/index.d.ts" }] ) ).toEqual({ ok: [new NotNeededPackage("ember__object", "@ember/object", "1.0.0")] }); diff --git a/packages/definitions-parser/test/packages.test.ts b/packages/definitions-parser/test/packages.test.ts index d8f670758f..411c05a00d 100644 --- a/packages/definitions-parser/test/packages.test.ts +++ b/packages/definitions-parser/test/packages.test.ts @@ -1,19 +1,17 @@ -import { createMockDT } from "../src/mocks"; +import { License } from "@definitelytyped/header-parser"; +import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; +import { Range } from "semver"; import { getTypingInfo } from "../src/lib/definition-parser"; +import { createMockDT } from "../src/mocks"; import { AllPackages, - TypingsVersions, - TypingsData, - getMangledNameForScopedPackage, NotNeededPackage, + TypingsData, + TypingsVersions, getDependencyFromFile, + getMangledNameForScopedPackage, } from "../src/packages"; -import { Range } from "semver"; -import { parseDefinitions } from "../src/parse-definitions"; -import { quietLoggerWithErrors } from "@definitelytyped/utils"; import { createTypingsVersionRaw } from "./utils"; -import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; -import { License } from "@definitelytyped/header-parser"; describe(AllPackages, () => { let allPackages: AllPackages; @@ -21,15 +19,16 @@ describe(AllPackages, () => { beforeAll(async () => { const dt = createMockDT(); dt.addOldVersionOfPackage("jquery", "1", "1.0.9999"); - const [log] = quietLoggerWithErrors(); - allPackages = await parseDefinitions(dt.fs, undefined, log); + allPackages = AllPackages.fromFS(dt.fs); }); - it("applies path mappings to test dependencies", () => { - const pkg = allPackages.tryGetLatestVersion("has-older-test-dependency")!; - expect(Array.from(allPackages.allDependencyTypings(pkg), ({ id }) => id)).toEqual([ - { typesDirectoryName: "jquery", version: { major: 1, minor: 0 } }, - ]); + it("applies path mappings to test dependencies", async () => { + const pkg = await allPackages.tryGetLatestVersion("has-older-test-dependency"); + for await (const { id } of allPackages.allDependencyTypings(pkg!)) { + expect(id).toEqual( + { typesDirectoryName: "jquery", version: { major: 1, minor: 0 } }, + ); + } }); describe("getNotNeededPackage", () => { @@ -76,7 +75,7 @@ describe(TypingsVersions, () => { if (Array.isArray(info)) { throw new Error(info.join("\n")); } - versions = new TypingsVersions(info); + versions = new TypingsVersions(dt.fs, info); }); it("sorts the data from latest to oldest version", () => { @@ -120,6 +119,14 @@ describe(TypingsData, () => { let data: TypingsData; beforeEach(() => { + const dt = createMockDT(); + dt.pkgDir("known") + .set("package.json", JSON.stringify({ + name: "@types/known" + })) + .set("index.d.ts", "declare const x: number;") + .set("tsconfig.json", "{}"); + const versions = createTypingsVersionRaw( "known", { @@ -129,7 +136,7 @@ describe(TypingsData, () => { "@types/known": "workspace:.", } ); - data = new TypingsData(versions["1.0"], true); + data = new TypingsData(dt.fs, versions["1.0"], true); }); it("sets the correct properties", () => { @@ -173,7 +180,7 @@ describe(TypingsData, () => { it("returns the versioned name if not latest", () => { const versions = createTypingsVersionRaw("known", {}, {}); - data = new TypingsData(versions["1.0"], false); + data = new TypingsData(createMockDT().fs, versions["1.0"], false); expect(data.desc).toBe("@types/known v1.0"); }); @@ -186,7 +193,7 @@ describe(TypingsData, () => { it("returns mangled name if scoped", () => { const versions = createTypingsVersionRaw("@foo/bar", {}, {}); - data = new TypingsData(versions["1.0"], false); + data = new TypingsData(createMockDT().fs, versions["1.0"], false); expect(data.typesDirectoryName).toBe("foo__bar"); }); diff --git a/packages/definitions-parser/test/parse-definitions.test.ts b/packages/definitions-parser/test/parse-definitions.test.ts deleted file mode 100644 index 2792e82a7b..0000000000 --- a/packages/definitions-parser/test/parse-definitions.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { createMockDT } from "../src/mocks"; -import { parseDefinitions } from "../src/parse-definitions"; -import { quietLoggerWithErrors } from "@definitelytyped/utils"; -import { testo } from "./utils"; - -testo({ - // async parseDefinitions() { - // const log = loggerWithErrors()[0] - // const dt = await getDefinitelyTyped(Options.defaults, log); - // const defs = await parseDefinitions(dt, undefined, log) - // expect(defs.allNotNeeded().length).toBeGreaterThan(0) - // expect(defs.allTypings().length).toBeGreaterThan(5000) - // const j = defs.tryGetLatestVersion("jquery") - // expect(j).toBeDefined() - // expect(j!.fullNpmName).toContain("types") - // expect(j!.fullNpmName).toContain("jquery") - // expect(defs.allPackages().length).toEqual(defs.allTypings().length + defs.allNotNeeded().length) - // }, - async mockParse() { - const log = quietLoggerWithErrors()[0]; - const defs = await parseDefinitions(createMockDT().fs, undefined, log); - expect(defs.allNotNeeded().length).toBe(1); - expect(defs.allTypings().length).toBe(6); - const j = defs.tryGetLatestVersion("jquery"); - expect(j).toBeDefined(); - expect(j!.name).toContain("types"); - expect(j!.name).toContain("jquery"); - expect(defs.allPackages().length).toEqual(defs.allTypings().length + defs.allNotNeeded().length); - }, -}); diff --git a/packages/definitions-parser/test/utils.ts b/packages/definitions-parser/test/utils.ts index 59806c785a..39eed617e2 100644 --- a/packages/definitions-parser/test/utils.ts +++ b/packages/definitions-parser/test/utils.ts @@ -24,12 +24,10 @@ export function createTypingsVersionRaw( nonNpm: false, projects: ["zombo.com"], }, - files: ["index.d.ts"], typesVersions: [], license: License.MIT, dependencies, devDependencies, - contentHash: "11111111111111", }, }; } diff --git a/packages/dtslint-runner/src/check-parse-results.ts b/packages/dtslint-runner/src/check-parse-results.ts index fdb45445da..e58c773e68 100644 --- a/packages/dtslint-runner/src/check-parse-results.ts +++ b/packages/dtslint-runner/src/check-parse-results.ts @@ -5,14 +5,14 @@ import * as semver from "semver"; if (require.main === module) { const options = { definitelyTypedPath: undefined, progress: false, parseInParallel: false }; getDefinitelyTyped(options, console).then((dt) => { - AllPackages.read(dt).then(checkParseResults); + return checkParseResults(AllPackages.fromFS(dt)); }); } -export function checkParseResults(allPackages: AllPackages): string[] { +export async function checkParseResults(allPackages: AllPackages): Promise { const errors = []; - for (const pkg of allPackages.allTypings()) { - for (const dep of allPackages.allDependencyTypings(pkg)) { + for (const pkg of await allPackages.allTypings()) { + for await (const dep of allPackages.allDependencyTypings(pkg)) { // check raw version because parsed version doesn't currently retain range information const version = assertDefined(new Map(pkg.allPackageJsonDependencies()).get(dep.name)); if (semver.parse(version)) { @@ -27,5 +27,5 @@ export function checkParseResults(allPackages: AllPackages): string[] { } } } - return errors; + return [...allPackages.getErrors(), ...errors]; } diff --git a/packages/dtslint-runner/src/prepareAffectedPackages.ts b/packages/dtslint-runner/src/prepareAffectedPackages.ts index 66b7b9e588..59925a53e4 100644 --- a/packages/dtslint-runner/src/prepareAffectedPackages.ts +++ b/packages/dtslint-runner/src/prepareAffectedPackages.ts @@ -1,8 +1,8 @@ import { getDefinitelyTyped, - parseDefinitions, getAffectedPackagesFromDiff, PreparePackagesResult, + AllPackages, } from "@definitelytyped/definitions-parser"; import { loggerWithErrors } from "@definitelytyped/utils"; import { checkParseResults } from "./check-parse-results"; @@ -18,16 +18,12 @@ export async function prepareAffectedPackages( parseInParallel: nProcesses > 1, }; const dt = await getDefinitelyTyped(options, log); - const allPackages = await parseDefinitions(dt, nProcesses ? { definitelyTypedPath, nProcesses } : undefined, log); - const errors = checkParseResults(allPackages); + const allPackages = AllPackages.fromFS(dt); + const checkErrors = await checkParseResults(allPackages); const result = await getAffectedPackagesFromDiff(allPackages, definitelyTypedPath); + const errors = [...checkErrors, ...(Array.isArray(result) ? result : [])]; if (errors.length) { throw new Error(errors.join("\n")); } - if (Array.isArray(result)) { - // TODO: This error handling doesn't make sense but matches the old way. Try removing the previous if statement - // and changing this one to if (errors.length || Array.isArray(result)) { ... } - throw new Error([...errors, ...result].join("\n")); - } - return result; + return result as PreparePackagesResult; } diff --git a/packages/dtslint-runner/src/prepareAllPackages.ts b/packages/dtslint-runner/src/prepareAllPackages.ts index a587ff7324..80d9ad1a54 100644 --- a/packages/dtslint-runner/src/prepareAllPackages.ts +++ b/packages/dtslint-runner/src/prepareAllPackages.ts @@ -1,4 +1,4 @@ -import { getDefinitelyTyped, parseDefinitions, PreparePackagesResult } from "@definitelytyped/definitions-parser"; +import { AllPackages, getDefinitelyTyped, PreparePackagesResult } from "@definitelytyped/definitions-parser"; import { execAndThrowErrors, loggerWithErrors, sleep } from "@definitelytyped/utils"; import { checkParseResults } from "./check-parse-results"; @@ -14,16 +14,16 @@ export async function prepareAllPackages( parseInParallel: nProcesses > 1, }; const dt = await getDefinitelyTyped(options, log); - const allPackages = await parseDefinitions(dt, nProcesses ? { definitelyTypedPath, nProcesses } : undefined, log); + const allPackages = AllPackages.fromFS(dt); if (clone) { await installAllDependencies(definitelyTypedPath); } - const errors = checkParseResults(allPackages); + const errors = await checkParseResults(allPackages); if (errors.length) { throw new Error(errors.join("\n")); } return { - packageNames: new Set(allPackages.allTypings().map(({ subDirectoryPath }) => subDirectoryPath)), + packageNames: new Set((await allPackages.allTypings()).map(({ subDirectoryPath }) => subDirectoryPath)), dependents: new Set(), }; } diff --git a/packages/publisher/package.json b/packages/publisher/package.json index d418bfe5ba..048a108bb8 100644 --- a/packages/publisher/package.json +++ b/packages/publisher/package.json @@ -34,7 +34,6 @@ "test": "../../node_modules/.bin/jest --config ../../jest.config.js packages/publisher", "clean": "node -r source-map-support/register dist/clean.js", "get-definitely-typed": "node -r source-map-support/register dist/get-definitely-typed.js", - "parse": "node -r source-map-support/register dist/parse-definitions.js", "calculate-versions": "node -r source-map-support/register dist/calculate-versions.js", "generate": "node -r source-map-support/register dist/generate-packages.js", "validate": "node -r source-map-support/register dist/validate.js", diff --git a/packages/publisher/src/calculate-versions.ts b/packages/publisher/src/calculate-versions.ts index cf0628d2ca..cae6f2ccb3 100644 --- a/packages/publisher/src/calculate-versions.ts +++ b/packages/publisher/src/calculate-versions.ts @@ -31,7 +31,7 @@ if (require.main === module) { export default async function calculateVersions(dt: FS, log: LoggerWithErrors): Promise { log.info("=== Calculating versions ==="); log.info("* Reading packages..."); - const packages = await AllPackages.read(dt); + const packages = AllPackages.fromFS(dt); return computeAndSaveChangedPackages(packages, log); } @@ -52,7 +52,7 @@ async function computeAndSaveChangedPackages( async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithErrors): Promise { log.info("# Computing changed packages..."); - const changedTypings = await nAtATime(npmRegistryParallelism, allPackages.allTypings(), async (pkg) => { + const changedTypings = await nAtATime(npmRegistryParallelism, await allPackages.allTypings(), async (pkg) => { const { version, needsPublish } = await fetchTypesPackageVersionInfo(pkg, /*publish*/ true, log); if (needsPublish) { log.info(`Need to publish: ${pkg.desc}@${version}`); @@ -70,7 +70,7 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE } const latestVersion = pkg.isLatest ? undefined - : (await fetchTypesPackageVersionInfo(allPackages.getLatest(pkg), /*publish*/ true)).version; + : (await fetchTypesPackageVersionInfo(await allPackages.getLatest(pkg), /*publish*/ true)).version; return { pkg, version, latestVersion }; } return undefined; diff --git a/packages/publisher/src/full.ts b/packages/publisher/src/full.ts index ba797a639b..82f64ed168 100644 --- a/packages/publisher/src/full.ts +++ b/packages/publisher/src/full.ts @@ -1,13 +1,12 @@ import * as yargs from "yargs"; +import { getDefinitelyTyped, ParseDefinitionsOptions } from "@definitelytyped/definitions-parser"; +import { Fetcher, loggerWithErrors, LoggerWithErrors, logUncaughtErrors } from "@definitelytyped/utils"; import calculateVersions from "./calculate-versions"; import { clean } from "./clean"; import generatePackages from "./generate-packages"; -import publishPackages from "./publish-packages"; -import { numberOfOsProcesses } from "./util/util"; -import { getDefinitelyTyped, parseDefinitions, ParseDefinitionsOptions } from "@definitelytyped/definitions-parser"; -import { Fetcher, assertDefined, logUncaughtErrors, loggerWithErrors, LoggerWithErrors } from "@definitelytyped/utils"; import { defaultLocalOptions } from "./lib/common"; +import publishPackages from "./publish-packages"; if (require.main === module) { const dry = !!yargs.argv.dry; @@ -25,13 +24,6 @@ export default async function full( ): Promise { clean(); const dt = await getDefinitelyTyped(options, log); - await parseDefinitions( - dt, - options.parseInParallel - ? { nProcesses: numberOfOsProcesses, definitelyTypedPath: assertDefined(options.definitelyTypedPath) } - : undefined, - log - ); const changedPackages = await calculateVersions(dt, log); await generatePackages(dt, changedPackages); await publishPackages(changedPackages, dry, githubAccessToken, fetcher); diff --git a/packages/publisher/src/generate-packages.ts b/packages/publisher/src/generate-packages.ts index d2a07413a3..0822473b91 100644 --- a/packages/publisher/src/generate-packages.ts +++ b/packages/publisher/src/generate-packages.ts @@ -41,7 +41,7 @@ if (require.main === module) { options.definitelyTypedPath = yargs.argv.path as string; } const dt = await getDefinitelyTyped(options, log); - const allPackages = await AllPackages.read(dt); + const allPackages = AllPackages.fromFS(dt); await generatePackages(dt, await readChangedPackages(allPackages), tgz); }); } diff --git a/packages/publisher/src/lib/versions.ts b/packages/publisher/src/lib/versions.ts index b569edc2b7..db88919c0a 100644 --- a/packages/publisher/src/lib/versions.ts +++ b/packages/publisher/src/lib/versions.ts @@ -37,11 +37,11 @@ export interface ChangedPackages { export async function readChangedPackages(allPackages: AllPackages): Promise { const json = (await readDataFile("calculate-versions", versionsFilename)) as ChangedPackagesJson; return { - changedTypings: json.changedTypings.map(({ id, version, latestVersion }) => ({ - pkg: allPackages.getTypingsData(id), + changedTypings: await Promise.all(json.changedTypings.map(async ({ id, version, latestVersion }) => ({ + pkg: await allPackages.getTypingsData(id), version, latestVersion, - })), + }))), changedNotNeededPackages: json.changedNotNeededPackages.map((id) => assertDefined(allPackages.getNotNeededPackage(id)) ), diff --git a/packages/publisher/src/parse-definitions.ts b/packages/publisher/src/parse-definitions.ts deleted file mode 100644 index 625d909da4..0000000000 --- a/packages/publisher/src/parse-definitions.ts +++ /dev/null @@ -1,46 +0,0 @@ -import os from "os"; -import process from "process"; -import yargs from "yargs"; -import { logUncaughtErrors, loggerWithErrors, assertDefined, FS } from "@definitelytyped/utils"; -import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common"; -import { getTypingInfo } from "@definitelytyped/definitions-parser/dist/lib/definition-parser"; -import { - getDefinitelyTyped, - parseDefinitions, - writeDataFile, - typesDataFilename, -} from "@definitelytyped/definitions-parser"; - -if (require.main === module) { - const { nProcesses, single: singleName } = yargs.options({ - single: { type: "string" }, - nProcesses: { type: "number" }, - }).argv; - - const options = process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions; - logUncaughtErrors(async () => { - const log = loggerWithErrors()[0]; - const dt = await getDefinitelyTyped(options, log); - if (singleName) { - await single(singleName, dt); - } else { - await parseDefinitions( - dt, - options.parseInParallel - ? { - nProcesses: nProcesses || os.cpus().length, - definitelyTypedPath: assertDefined(options.definitelyTypedPath), - } - : undefined, - log - ); - } - }); -} - -async function single(singleName: string, dt: FS): Promise { - const data = await getTypingInfo(singleName, dt); - const typings = { [singleName]: data }; - await writeDataFile(typesDataFilename, typings); - console.log(JSON.stringify(data, undefined, 4)); -} diff --git a/packages/publisher/src/publish-packages.ts b/packages/publisher/src/publish-packages.ts index 2c40fc7afa..8f8e9336b6 100644 --- a/packages/publisher/src/publish-packages.ts +++ b/packages/publisher/src/publish-packages.ts @@ -24,7 +24,7 @@ if (require.main === module) { } const dt = await getDefinitelyTyped(options, loggerWithErrors()[0]); await publishPackages( - await readChangedPackages(await AllPackages.read(dt)), + await readChangedPackages(AllPackages.fromFS(dt)), dry, process.env.GH_API_TOKEN || "", new Fetcher() diff --git a/packages/publisher/src/publish-registry.ts b/packages/publisher/src/publish-registry.ts index 2dc7e3d2b7..2f4156e9dc 100644 --- a/packages/publisher/src/publish-registry.ts +++ b/packages/publisher/src/publish-registry.ts @@ -47,7 +47,7 @@ if (require.main === module) { process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, loggerWithErrors()[0] ); - await publishRegistry(dt, await AllPackages.read(dt), dry); + await publishRegistry(dt, AllPackages.fromFS(dt), dry); }); } @@ -59,7 +59,7 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages, assert(semver.satisfies(latestVersion, "~0.1")); // Don't include not-needed packages in the registry. - const registryJsonData = await generateRegistry(allPackages.allLatestTypings()); + const registryJsonData = await generateRegistry(await allPackages.allLatestTypings()); const registry = JSON.stringify(registryJsonData); const newContentHash = computeHash(registry); const newVersion = semver.inc(latestVersion, "patch")!; diff --git a/packages/publisher/test/generate-packages.test.ts b/packages/publisher/test/generate-packages.test.ts index 7f85579aab..0811f82dab 100644 --- a/packages/publisher/test/generate-packages.test.ts +++ b/packages/publisher/test/generate-packages.test.ts @@ -5,7 +5,7 @@ import { getLicenseFileText, } from "../src/generate-packages"; import { License } from "@definitelytyped/header-parser"; -import { NotNeededPackage, TypingsData, TypingsDataRaw } from "@definitelytyped/definitions-parser"; +import { NotNeededPackage, TypingsData, TypingsDataRaw, createMockDT } from "@definitelytyped/definitions-parser"; import { testo } from "./utils"; import { InMemoryFS, Dir, FS } from "@definitelytyped/utils"; @@ -24,11 +24,9 @@ function createRawPackage(license: License): TypingsDataRaw { nonNpm: false, }, typesVersions: [], - files: ["index.d.ts", "jquery.test.ts"], license, dependencies: { "@types/madeira": "^1", balzac: "~3" }, devDependencies: { "@types/jquery": "workspace:." }, - contentHash: "11", }; } @@ -50,37 +48,37 @@ function defaultFS(): FS { testo({ mitLicenseText() { - const typing = new TypingsData(createRawPackage(License.MIT), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.MIT), /*isLatest*/ true); expect(getLicenseFileText(typing)).toEqual(expect.stringContaining("MIT License")); }, apacheLicenseText() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); expect(getLicenseFileText(typing)).toEqual(expect.stringContaining("Apache License, Version 2.0")); }, basicReadme() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); expect(createReadme(typing, defaultFS())).toEqual( expect.stringContaining("This package contains type definitions for") ); }, readmeContainsContributors() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); expect(createReadme(typing, defaultFS())).toEqual( expect.stringContaining("written by [A](b@c.d), and [E](https://github.com/e)") ); }, readmeContainsProjectName() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); expect(createReadme(typing, defaultFS())).toEqual(expect.stringContaining("jquery.org")); }, readmeOneDependency() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); expect(createReadme(typing, defaultFS())).toEqual( expect.stringContaining("Dependencies: [@types/madeira](https://npmjs.com/package/@types/madeira)") ); }, readmeMultipleDependencies() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); typing.dependencies["@types/example"] = "*"; expect(createReadme(typing, defaultFS())).toEqual( expect.stringContaining( @@ -89,18 +87,20 @@ testo({ ); }, readmeContainsSingleFileDTS() { - const typing = new TypingsData(createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); expect(createReadme(typing, defaultFS())).toContain("type T = import"); }, readmeContainsManyDTSFilesDoesNotAmendREADME() { const rawPkg = createRawPackage(License.Apache20); - // @ts-expect-error - files is readonly - rawPkg.files = ["index.d.ts", "other.d.ts"]; - const typing = new TypingsData(rawPkg, /*isLatest*/ true); - expect(createReadme(typing, defaultFS())).not.toContain("type T = import"); + const dt = createMockDT(); + dt.pkgDir("jquery") + .set("index.d.ts", `type T = import("./types");`) + .set("other.d.ts", ""); + const typing = new TypingsData(dt.fs, rawPkg, /*isLatest*/ true); + expect(createReadme(typing, dt.fs)).not.toContain("type T = import"); }, basicPackageJson() { - const typing = new TypingsData(createRawPackage(License.MIT), /*isLatest*/ true); + const typing = new TypingsData(defaultFS(), createRawPackage(License.MIT), /*isLatest*/ true); expect(createPackageJSON(typing, "1.0")).toEqual(`{ "name": "@types/jquery", "version": "1.0", diff --git a/packages/retag/src/index.ts b/packages/retag/src/index.ts index 495dde5e73..6fda41bb25 100644 --- a/packages/retag/src/index.ts +++ b/packages/retag/src/index.ts @@ -20,7 +20,6 @@ import { AnyPackage, TypingsData, AllPackages, - parseDefinitions, getDefinitelyTyped, } from "@definitelytyped/definitions-parser"; import * as pacote from "pacote"; @@ -47,25 +46,21 @@ async function main() { * This shouldn't normally need to run, since we run `tagSingle` whenever we publish a package. * But this should be run if the way we calculate tags changes (e.g. when a new release is allowed to be tagged "latest"). */ -async function tag(dry: boolean, nProcesses: number, name?: string) { +async function tag(dry: boolean, _nProcesses: number, name?: string) { const log = loggerWithErrors()[0]; const options = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true }; - await parseDefinitions( - await getDefinitelyTyped(options, log), - { nProcesses: nProcesses || os.cpus().length, definitelyTypedPath: "../DefinitelyTyped" }, - log - ); - + const dt = await getDefinitelyTyped(options, log); const token = process.env.NPM_TOKEN as string; const publishClient = await NpmPublishClient.create(token, {}); if (name) { - const pkg = await AllPackages.readSingle(name); + const pkg = await AllPackages.readSingle(dt,name); const version = await getLatestTypingVersion(pkg); await updateTypeScriptVersionTags(pkg, version, publishClient, consoleLogger.info, dry); await updateLatestTag(pkg.name, version, publishClient, consoleLogger.info, dry); } else { - await nAtATime(5, await AllPackages.readLatestTypings(), async (pkg) => { + const allPackages = AllPackages.fromFS(dt); + await nAtATime(5, await allPackages.allLatestTypings(), async (pkg) => { // Only update tags for the latest version of the package. const version = await getLatestTypingVersion(pkg); await updateTypeScriptVersionTags(pkg, version, publishClient, consoleLogger.info, dry); From 02cea84675f52484db3451bd19e0374982eb3d27 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 18 Oct 2023 14:36:21 -0700 Subject: [PATCH 02/12] Fix publisher tests --- .../src/lib/definition-parser.ts | 2 +- packages/definitions-parser/src/packages.ts | 2 +- .../publisher/test/generate-packages.test.ts | 67 ++++++++++--------- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/definitions-parser/src/lib/definition-parser.ts b/packages/definitions-parser/src/lib/definition-parser.ts index 6897e261f8..772cfedfbb 100644 --- a/packages/definitions-parser/src/lib/definition-parser.ts +++ b/packages/definitions-parser/src/lib/definition-parser.ts @@ -391,7 +391,7 @@ function getFilesForSingleTypeScriptVersion( return { typescriptVersion, declFiles: typescriptVersion === undefined ? declFiles : declFiles.map(f => `ts${typescriptVersion}/${f}`), - tsconfigPathsForHash: JSON.stringify(tsconfig.compilerOptions.paths), + tsconfigPathsForHash: JSON.stringify(tsconfig.compilerOptions?.paths), }; } diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index a05ed6258f..2db497ea26 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -506,7 +506,7 @@ export class TypingsData extends PackageBase { return this._contentHash ??= hash( [...this.files, "package.json"], mapDefined(this.typesVersionsFiles!, (a) => a.tsconfigPathsForHash), - this.dt + this.dt.subDir("types").subDir(this.typesDirectoryName) ); } get projectName(): string | undefined { diff --git a/packages/publisher/test/generate-packages.test.ts b/packages/publisher/test/generate-packages.test.ts index 0811f82dab..feaa8ba225 100644 --- a/packages/publisher/test/generate-packages.test.ts +++ b/packages/publisher/test/generate-packages.test.ts @@ -1,13 +1,12 @@ +import { DTMock, NotNeededPackage, TypingsData, TypingsDataRaw } from "@definitelytyped/definitions-parser"; +import { License } from "@definitelytyped/header-parser"; import { createNotNeededPackageJSON, createPackageJSON, createReadme, getLicenseFileText, } from "../src/generate-packages"; -import { License } from "@definitelytyped/header-parser"; -import { NotNeededPackage, TypingsData, TypingsDataRaw, createMockDT } from "@definitelytyped/definitions-parser"; import { testo } from "./utils"; -import { InMemoryFS, Dir, FS } from "@definitelytyped/utils"; function createRawPackage(license: License): TypingsDataRaw { return { @@ -34,73 +33,75 @@ function createUnneededPackage() { return new NotNeededPackage("absalom", "alternate", "1.1.1"); } -function defaultFS(): FS { - const pkg = new Dir(undefined); - pkg.set( - "index.d.ts", - `type T = import("./types"); -` - ); - pkg.set("jquery.test.ts", "// tests"); - const memFS = new InMemoryFS(pkg, "/types/mock/"); - return memFS; +function defaultFS() { + const dt = new DTMock(); + dt.pkgDir("jquery") + .set("package.json", JSON.stringify({ name: "@types/jquery" })) + .set("tsconfig.json", `{ "files": ["index.d.ts", "jquery-tests.ts"] }`) + .set("index.d.ts",`type T = import("./types");\n`) + .set("jquery-tests.ts", "// tests"); + return dt; } testo({ mitLicenseText() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.MIT), /*isLatest*/ true); + const typing = new TypingsData(defaultFS().fs, createRawPackage(License.MIT), /*isLatest*/ true); expect(getLicenseFileText(typing)).toEqual(expect.stringContaining("MIT License")); }, apacheLicenseText() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); + const typing = new TypingsData(defaultFS().fs, createRawPackage(License.Apache20), /*isLatest*/ true); expect(getLicenseFileText(typing)).toEqual(expect.stringContaining("Apache License, Version 2.0")); }, basicReadme() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); - expect(createReadme(typing, defaultFS())).toEqual( + const dt = defaultFS(); + const typing = new TypingsData(dt.fs, createRawPackage(License.Apache20), /*isLatest*/ true); + expect(createReadme(typing, dt.pkgFS("jquery"))).toEqual( expect.stringContaining("This package contains type definitions for") ); }, readmeContainsContributors() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); - expect(createReadme(typing, defaultFS())).toEqual( + const dt = defaultFS(); + const typing = new TypingsData(dt.fs, createRawPackage(License.Apache20), /*isLatest*/ true); + expect(createReadme(typing, dt.pkgFS("jquery"))).toEqual( expect.stringContaining("written by [A](b@c.d), and [E](https://github.com/e)") ); }, readmeContainsProjectName() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); - expect(createReadme(typing, defaultFS())).toEqual(expect.stringContaining("jquery.org")); + const dt = defaultFS(); + const typing = new TypingsData(dt.fs, createRawPackage(License.Apache20), /*isLatest*/ true); + expect(createReadme(typing, dt.pkgFS("jquery"))).toEqual(expect.stringContaining("jquery.org")); }, readmeOneDependency() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); - expect(createReadme(typing, defaultFS())).toEqual( + const dt = defaultFS(); + const typing = new TypingsData(dt.fs, createRawPackage(License.Apache20), /*isLatest*/ true); + expect(createReadme(typing, dt.pkgFS("jquery"))).toEqual( expect.stringContaining("Dependencies: [@types/madeira](https://npmjs.com/package/@types/madeira)") ); }, readmeMultipleDependencies() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); + const dt = defaultFS(); + const typing = new TypingsData(dt.fs, createRawPackage(License.Apache20), /*isLatest*/ true); typing.dependencies["@types/example"] = "*"; - expect(createReadme(typing, defaultFS())).toEqual( + expect(createReadme(typing, dt.pkgFS("jquery"))).toEqual( expect.stringContaining( "Dependencies: [@types/example](https://npmjs.com/package/@types/example), [@types/madeira](https://npmjs.com/package/@types/madeira)" ) ); }, readmeContainsSingleFileDTS() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.Apache20), /*isLatest*/ true); - expect(createReadme(typing, defaultFS())).toContain("type T = import"); + const dt = defaultFS(); + const typing = new TypingsData(dt.fs, createRawPackage(License.Apache20), /*isLatest*/ true); + expect(createReadme(typing, dt.pkgFS("jquery"))).toContain("type T = import"); }, readmeContainsManyDTSFilesDoesNotAmendREADME() { const rawPkg = createRawPackage(License.Apache20); - const dt = createMockDT(); - dt.pkgDir("jquery") - .set("index.d.ts", `type T = import("./types");`) - .set("other.d.ts", ""); + const dt = defaultFS(); + dt.pkgDir("jquery").set("other.d.ts", ""); const typing = new TypingsData(dt.fs, rawPkg, /*isLatest*/ true); expect(createReadme(typing, dt.fs)).not.toContain("type T = import"); }, basicPackageJson() { - const typing = new TypingsData(defaultFS(), createRawPackage(License.MIT), /*isLatest*/ true); + const typing = new TypingsData(defaultFS().fs, createRawPackage(License.MIT), /*isLatest*/ true); expect(createPackageJSON(typing, "1.0")).toEqual(`{ "name": "@types/jquery", "version": "1.0", @@ -129,7 +130,7 @@ testo({ "@types/madeira": "^1", "balzac": "~3" }, - "typesPublisherContentHash": "11", + "typesPublisherContentHash": "9d3169349ad27006640a88a02cfb8269edfb00b3bcf73acc7cd133d019c6e154", "typeScriptVersion": "4.5" }`); }, From 86c06c92e60cdd6c7e767417a0aa760e00ec2ad0 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 18 Oct 2023 16:08:18 -0700 Subject: [PATCH 03/12] Fix other tests --- packages/definitions-parser/src/git.ts | 8 ++-- .../src/lib/definition-parser.ts | 31 +++++++++---- packages/definitions-parser/src/packages.ts | 21 +++++++-- .../test/definition-parser.test.ts | 44 ++++++++++--------- packages/definitions-parser/test/git.test.ts | 24 +++++----- .../definitions-parser/test/packages.test.ts | 14 +++--- 6 files changed, 86 insertions(+), 56 deletions(-) diff --git a/packages/definitions-parser/src/git.ts b/packages/definitions-parser/src/git.ts index c4bedbbd43..92e37b3f3e 100644 --- a/packages/definitions-parser/src/git.ts +++ b/packages/definitions-parser/src/git.ts @@ -86,7 +86,7 @@ export async function getAffectedPackagesFromDiff( const errors = []; const diffs = await gitDiff(consoleLogger.info, definitelyTypedPath); if (diffs.find((d) => d.file === "notNeededPackages.json")) { - const deleteds = getNotNeededPackages(allPackages, diffs); + const deleteds = await getNotNeededPackages(allPackages, diffs); if ("errors" in deleteds) errors.push(...deleteds.errors); else for (const deleted of deleteds.ok) { @@ -144,17 +144,17 @@ it is supposed to replace, ${typings.version} of ${unneeded.name}.`); * 1. Find all the deleted files and group by package (error on deleted files outside a package). * 2. Make sure that all deleted packages in notNeededPackages have no files left. */ -export function getNotNeededPackages( +export async function getNotNeededPackages( allPackages: AllPackages, diffs: GitDiff[] -): { errors: string[] } | { ok: NotNeededPackage[] } { +): Promise<{ errors: string[] } | { ok: NotNeededPackage[] }> { const deletions = gitDeletions(diffs); if ("errors" in deletions) return deletions; const deletedPackages = new Set(deletions.ok.map((p) => assertDefined(p.typesDirectoryName))); const notNeededs = []; const errors = []; for (const p of deletedPackages) { - const hasTyping = allPackages.hasTypingFor({ typesDirectoryName: p, version: "*" }); + const hasTyping = await allPackages.hasTypingFor({ typesDirectoryName: p, version: "*" }); const notNeeded = allPackages.getNotNeededPackage(p); if (hasTyping && notNeeded) { errors.push(`Please delete all files in ${p} when adding it to notNeededPackages.json.`); diff --git a/packages/definitions-parser/src/lib/definition-parser.ts b/packages/definitions-parser/src/lib/definition-parser.ts index 772cfedfbb..135ff96c94 100644 --- a/packages/definitions-parser/src/lib/definition-parser.ts +++ b/packages/definitions-parser/src/lib/definition-parser.ts @@ -50,7 +50,7 @@ function formattedLibraryVersion(typingsDataRaw: TypingsDataRaw): `${number}.${n return `${typingsDataRaw.header.libraryMajorVersion}.${typingsDataRaw.header.libraryMinorVersion}`; } -export async function getTypingInfo(packageNameOrTypesDirectoryName: string, dt: FS): Promise { +export async function getTypingInfo(packageNameOrTypesDirectoryName: string, dt: FS): Promise { const errors = []; if (packageNameOrTypesDirectoryName !== packageNameOrTypesDirectoryName.toLowerCase()) { errors.push(`Package name \`${packageNameOrTypesDirectoryName}\` should be strictly lowercase`); @@ -60,7 +60,12 @@ export async function getTypingInfo(packageNameOrTypesDirectoryName: string, dt: readonly version: DirectoryParsedTypingVersion; } - const fs = dt.subDir("types").subDir(getMangledNameForScopedPackage(packageNameOrTypesDirectoryName)); + const typesDirectoryName = getMangledNameForScopedPackage(packageNameOrTypesDirectoryName); + if (!dt.subDir("types").exists(typesDirectoryName) || !dt.subDir("types").isDirectory(typesDirectoryName)) { + return undefined; + } + + const fs = dt.subDir("types").subDir(typesDirectoryName); const [rootDirectoryLs, olderVersionDirectories] = split( fs.readdir(), (fileOrDirectoryName) => { @@ -338,13 +343,21 @@ function getFilesForSingleTypeScriptVersion( path.resolve("/", fs.debugPath()) ).options; errors.push(...checkFilesFromTsConfig(typesDirectoryName, tsconfig, fs.debugPath())); - const { types, tests } = allReferencedFiles( - tsconfig.files ?? [], - fs, - typesDirectoryName, - moduleResolutionHost, - compilerOptions - ); + let types, tests; + try { + ({ types, tests } = allReferencedFiles( + tsconfig.files ?? [], + fs, + typesDirectoryName, + moduleResolutionHost, + compilerOptions + )); + } catch (err) { + if (err instanceof Error) { + errors.push(err.message); + } + return errors; + } const usedFiles = new Set( [...types.keys(), ...tests, "tsconfig.json", "tslint.json"].map((f) => slicePrefixes(f, "node_modules/@types/" + typesDirectoryName + "/") diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index 2db497ea26..970193da2f 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -13,7 +13,9 @@ export class AllPackages { } static fromTestData(typingsVersionsRaw: Record, notNeeded: readonly NotNeededPackage[]) { - const fs = new InMemoryFS(new Dir(/*parent*/ undefined), ""); + const rootDir = new Dir(/*parent*/ undefined); + rootDir.set("types", new Dir(/*parent*/ rootDir)); + const fs = new InMemoryFS(rootDir, "/"); return new AllPackages( fs, new Map(Object.entries(typingsVersionsRaw).map(([name, raw]) => [name, new TypingsVersions(fs, raw)])), @@ -24,9 +26,12 @@ export class AllPackages { /** Use for `--single` tasks only. Do *not* call this in a loop! */ static async readSingle(dt: FS, name: string): Promise { const data = await getTypingInfo(name, dt); - if ("errors" in data) { + if (!data) { throw new Error(`Can't find package ${name}`); } + if ("errors" in data) { + throw new Error(`Errors while reading package ${name}: ${data.errors.join("\n")}`); + } const versions = Object.values(data); if (versions.length > 1) { throw new Error(`Package ${name} has multiple versions.`); @@ -49,8 +54,8 @@ export class AllPackages { return this.notNeeded.find((p) => p.typesDirectoryName === typesDirectoryName); } - hasTypingFor(dep: PackageId): boolean { - return this.tryGetTypingsData(dep) !== undefined; + async hasTypingFor(dep: PackageId): Promise { + return await this.tryGetTypingsData(dep) !== undefined; } getErrors() { @@ -127,6 +132,9 @@ export class AllPackages { return undefined; } const raw = await getTypingInfo(typesDirectoryName, this.dt); + if (!raw) { + return undefined; + } if ("errors" in raw) { this.errors.set(typesDirectoryName, raw.errors); return undefined; @@ -466,6 +474,11 @@ export class TypingsData extends PackageBase { get typesVersions(): readonly TypeScriptVersion[] { return this.data.typesVersions; } + + getErrors() { + void this.files; + return this.errors; + } private typesVersionsFiles: readonly FilesForSingleTypeScriptVersion[] | undefined; get files(): readonly string[] { diff --git a/packages/definitions-parser/test/definition-parser.test.ts b/packages/definitions-parser/test/definition-parser.test.ts index 9c48a58d7f..432c8eb629 100644 --- a/packages/definitions-parser/test/definition-parser.test.ts +++ b/packages/definitions-parser/test/definition-parser.test.ts @@ -1,7 +1,8 @@ -import path from "path"; import { DiskFS } from "@definitelytyped/utils"; -import { createMockDT } from "../src/mocks"; +import path from "path"; import { getTypingInfo } from "../src/lib/definition-parser"; +import { createMockDT } from "../src/mocks"; +import { TypingsVersions } from "../src/packages"; describe(getTypingInfo, () => { it("keys data by major.minor version", async () => { @@ -10,13 +11,13 @@ describe(getTypingInfo, () => { dt.addOldVersionOfPackage("jquery", "2", "2.0.9999"); const info = await getTypingInfo("jquery", dt.fs); - expect(Object.keys(info).sort()).toEqual(["1.42", "2.0", "3.3"]); + expect(Object.keys(info!).sort()).toEqual(["1.42", "2.0", "3.3"]); }); it("works for a package with dependencies", async () => { const dt = createMockDT(); const info = await getTypingInfo("has-dependency", dt.fs); - expect("errors" in info).toBeFalsy(); + expect("errors" in info!).toBeFalsy(); }); it("works for non-module files with empty statements", async () => { @@ -51,7 +52,7 @@ describe(getTypingInfo, () => { ); const info = await getTypingInfo("example", dt.fs); - expect("errors" in info).toBeFalsy(); + expect("errors" in info!).toBeFalsy(); }); it("works for a scoped package with scoped older dependencies", async () => { const dt = createMockDT(); @@ -123,7 +124,7 @@ export function myFunction(arg:string): string; dt.addOldVersionOfPackage("@ckeditor/ckeditor5-utils", "10", "10.0.9999"); const info = await getTypingInfo("ckeditor__ckeditor5-engine", dt.fs); - expect("errors" in info).toBeFalsy(); + expect("errors" in info!).toBeFalsy(); }); it("allows path mapping to node/buffer", async () => { @@ -192,11 +193,11 @@ export * from 'buffer'; ); const info = await getTypingInfo("safer", dt.fs); - if ("errors" in info) { + if ("errors" in info!) { throw new Error(info.errors.join("\n")); } - expect(info["1.0"]).toBeDefined(); - expect(info["1.0"].dependencies).toEqual({ "@types/node": "*" }); + expect(info!["1.0"]).toBeDefined(); + expect(info!["1.0"].dependencies).toEqual({ "@types/node": "*" }); }); it("errors on arbitrary path mapping", () => {}); it("supports node_modules passthrough path mapping", async () => { @@ -277,7 +278,7 @@ const a = new webpack.AutomaticPrefetchPlugin(); ); const info = await getTypingInfo("webpack", dt.fs); - expect("errors" in info).toBeFalsy(); + expect("errors" in info!).toBeFalsy(); }); it("allows references to old versions of self", async () => { @@ -285,7 +286,7 @@ const a = new webpack.AutomaticPrefetchPlugin(); "fail", new DiskFS(path.resolve(__dirname, "fixtures/allows-references-to-old-versions-of-self/")) ); - expect("errors" in info).toBeFalsy(); + expect("errors" in info!).toBeFalsy(); }); it("omits test dependencies on modules declared in index.d.ts", async () => { @@ -355,7 +356,7 @@ import route = require('@ember/routing/route'); }` ); - const info = await getTypingInfo("ember", dt.fs); + const info = (await getTypingInfo("ember", dt.fs))!; if ("errors" in info) { throw new Error(info.errors.join("\n")); } @@ -363,10 +364,10 @@ import route = require('@ember/routing/route'); }); it("doesn't omit dependencies if only some deep modules are declared", async () => { - const info = await getTypingInfo( + const info = (await getTypingInfo( "styled-components-react-native", new DiskFS(path.resolve(__dirname, "fixtures/doesnt-omit-dependencies-if-only-some-deep-modules-are-declared/")) - ); + ))!; if ("errors" in info) { throw new Error(info.errors.join("\n")); } @@ -374,12 +375,15 @@ import route = require('@ember/routing/route'); }); it("rejects relative references to other packages", async () => { - expect(() => - getTypingInfo( - "referencing", - new DiskFS(path.resolve(__dirname, "fixtures/rejects-relative-references-to-other-packages/")) - ) - ).rejects.toThrow("Definitions must use global references to other packages"); + const dt = new DiskFS(path.resolve(__dirname, "fixtures/rejects-relative-references-to-other-packages/")); + const raw = (await getTypingInfo("referencing", dt))!; + if ("errors" in raw) { + throw new Error(raw.errors.join("\n")); + } + const typingData = new TypingsVersions(dt, raw).getLatest(); + const errors = typingData.getErrors(); + expect(errors).toHaveLength(1); + expect(errors![0]).toMatch("Definitions must use global references to other packages"); }); describe("concerning multiple versions", () => { diff --git a/packages/definitions-parser/test/git.test.ts b/packages/definitions-parser/test/git.test.ts index 872cd80039..740f16581b 100644 --- a/packages/definitions-parser/test/git.test.ts +++ b/packages/definitions-parser/test/git.test.ts @@ -23,33 +23,33 @@ const deleteJestDiffs: GitDiff[] = [ ]; testo({ - ok() { - expect(getNotNeededPackages(allPackages, deleteJestDiffs)).toEqual({ ok: jestNotNeeded }); + async ok() { + expect(await getNotNeededPackages(allPackages, deleteJestDiffs)).toEqual({ ok: jestNotNeeded }); }, - forgotToDeleteFiles() { + async forgotToDeleteFiles() { expect( - getNotNeededPackages( + await getNotNeededPackages( AllPackages.fromTestData({ jest: createTypingsVersionRaw("jest", {}, {}) }, jestNotNeeded), deleteJestDiffs ) ).toEqual({ errors: ["Please delete all files in jest when adding it to notNeededPackages.json."] }); }, - tooManyDeletes() { - expect(getNotNeededPackages(allPackages, [{ status: "D", file: "oops.txt" }])).toEqual({ + async tooManyDeletes() { + expect(await getNotNeededPackages(allPackages, [{ status: "D", file: "oops.txt" }])).toEqual({ errors: [ `Unexpected file deleted: oops.txt When removing packages, you should only delete files that are a part of removed packages.`, ], }); }, - deleteInOtherPackage() { + async deleteInOtherPackage() { expect( - getNotNeededPackages(allPackages, [...deleteJestDiffs, { status: "D", file: "types/most-recent/extra-tests.ts" }]) + await getNotNeededPackages(allPackages, [...deleteJestDiffs, { status: "D", file: "types/most-recent/extra-tests.ts" }]) ).toEqual({ ok: jestNotNeeded }); }, - extraneousFile() { + async extraneousFile() { expect( - getNotNeededPackages(allPackages, [ + await getNotNeededPackages(allPackages, [ { status: "A", file: "oooooooooooops.txt" }, { status: "M", file: "notNeededPackages.json" }, { status: "D", file: "types/jest/index.d.ts" }, @@ -57,9 +57,9 @@ When removing packages, you should only delete files that are a part of removed ]) ).toEqual({ ok: jestNotNeeded }); }, - scoped() { + async scoped() { expect( - getNotNeededPackages( + await getNotNeededPackages( AllPackages.fromTestData(typesData, [new NotNeededPackage("ember__object", "@ember/object", "1.0.0")]), [{ status: "D", file: "types/ember__object/index.d.ts" }] ) diff --git a/packages/definitions-parser/test/packages.test.ts b/packages/definitions-parser/test/packages.test.ts index 411c05a00d..f6472ad4ba 100644 --- a/packages/definitions-parser/test/packages.test.ts +++ b/packages/definitions-parser/test/packages.test.ts @@ -40,21 +40,21 @@ describe(AllPackages, () => { }); describe("hasTypingFor", () => { - it("returns true if typings exist", () => { + it("returns true if typings exist", async () => { expect( - allPackages.hasTypingFor({ + await allPackages.hasTypingFor({ name: "@types/jquery", version: "*", }) ).toBe(true); expect( - allPackages.hasTypingFor({ + await allPackages.hasTypingFor({ typesDirectoryName: "jquery", version: "*", }) ).toBe(true); expect( - allPackages.hasTypingFor({ + await allPackages.hasTypingFor({ name: "@types/nonExistent", version: "*", }) @@ -75,7 +75,7 @@ describe(TypingsVersions, () => { if (Array.isArray(info)) { throw new Error(info.join("\n")); } - versions = new TypingsVersions(dt.fs, info); + versions = new TypingsVersions(dt.fs, info!); }); it("sorts the data from latest to oldest version", () => { @@ -125,7 +125,7 @@ describe(TypingsData, () => { name: "@types/known" })) .set("index.d.ts", "declare const x: number;") - .set("tsconfig.json", "{}"); + .set("tsconfig.json", `{ "files": ["index.d.ts"] }`); const versions = createTypingsVersionRaw( "known", @@ -155,7 +155,7 @@ describe(TypingsData, () => { expect(data.typesVersions).toEqual([]); expect(data.files).toEqual(["index.d.ts"]); expect(data.license).toBe(License.MIT); - expect(data.contentHash).toBe("11111111111111"); + expect(data.contentHash).toBe("f647d34b5793cea752bc5b892d2099c92f1ced5f13b8a4ec3e4826d9f9cd0163"); expect(data.projectName).toBe("zombo.com"); expect(data.dependencies).toEqual({ "dependency-1": "*", From fc31a44fe52bc2522ed4bceb414d649a7a256170 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 18 Oct 2023 16:27:03 -0700 Subject: [PATCH 04/12] Ensure file errors are handled --- packages/definitions-parser/src/packages.ts | 31 ++++++++++--------- .../test/definition-parser.test.ts | 9 ++++-- .../definitions-parser/test/packages.test.ts | 2 +- packages/publisher/src/generate-packages.ts | 9 +++--- .../publisher/test/generate-packages.test.ts | 11 ++++++- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index 970193da2f..15dc4398a5 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -442,8 +442,6 @@ export class TypingsVersions { } export class TypingsData extends PackageBase { - public errors: readonly string[] | undefined; - constructor(private dt: FS, private readonly data: TypingsDataRaw, readonly isLatest: boolean, private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath())) { super(); } @@ -474,28 +472,31 @@ export class TypingsData extends PackageBase { get typesVersions(): readonly TypeScriptVersion[] { return this.data.typesVersions; } - - getErrors() { - void this.files; - return this.errors; - } private typesVersionsFiles: readonly FilesForSingleTypeScriptVersion[] | undefined; - get files(): readonly string[] { + tryGetFiles(): readonly string[] | { errors: string[] } { if (!this.typesVersionsFiles) { const files = getFiles(this.dt, this, this.moduleResolutionHost); if ("errors" in files) { - this.errors = files.errors; - this.typesVersionsFiles = []; - } else { - this.typesVersionsFiles = files; + return files; } + this.typesVersionsFiles = files; } return this.typesVersionsFiles.flatMap((v) => v.declFiles); } - get dtsFiles(): readonly string[] { - return this.files.filter((f) => f.endsWith(".d.ts") || f.endsWith(".d.mts") || f.endsWith(".d.cts")); + + getFiles(): readonly string[] { + const files = this.tryGetFiles(); + if ("errors" in files) { + throw new Error(`Errors while reading package ${this.name}: ${files.errors.join("\n")}`); + } + return files; + } + + getDtsFiles(): readonly string[] { + return this.getFiles().filter((f) => f.endsWith(".d.ts") || f.endsWith(".d.mts") || f.endsWith(".d.cts")); } + get license(): License { return this.data.license; } @@ -517,7 +518,7 @@ export class TypingsData extends PackageBase { private _contentHash: string | undefined; get contentHash(): string { return this._contentHash ??= hash( - [...this.files, "package.json"], + [...this.getFiles(), "package.json"], mapDefined(this.typesVersionsFiles!, (a) => a.tsconfigPathsForHash), this.dt.subDir("types").subDir(this.typesDirectoryName) ); diff --git a/packages/definitions-parser/test/definition-parser.test.ts b/packages/definitions-parser/test/definition-parser.test.ts index 432c8eb629..3867edd6b2 100644 --- a/packages/definitions-parser/test/definition-parser.test.ts +++ b/packages/definitions-parser/test/definition-parser.test.ts @@ -381,9 +381,12 @@ import route = require('@ember/routing/route'); throw new Error(raw.errors.join("\n")); } const typingData = new TypingsVersions(dt, raw).getLatest(); - const errors = typingData.getErrors(); - expect(errors).toHaveLength(1); - expect(errors![0]).toMatch("Definitions must use global references to other packages"); + const files = typingData.tryGetFiles(); + if (!("errors" in files)) { + throw new Error("Expected errors"); + } + expect(files.errors).toHaveLength(1); + expect(files.errors[0]).toMatch("Definitions must use global references to other packages"); }); describe("concerning multiple versions", () => { diff --git a/packages/definitions-parser/test/packages.test.ts b/packages/definitions-parser/test/packages.test.ts index f6472ad4ba..c8adb499b5 100644 --- a/packages/definitions-parser/test/packages.test.ts +++ b/packages/definitions-parser/test/packages.test.ts @@ -153,7 +153,7 @@ describe(TypingsData, () => { expect(data.minor).toBe(0); expect(data.minTypeScriptVersion).toBe(TypeScriptVersion.lowest); expect(data.typesVersions).toEqual([]); - expect(data.files).toEqual(["index.d.ts"]); + expect(data.getFiles()).toEqual(["index.d.ts"]); expect(data.license).toBe(License.MIT); expect(data.contentHash).toBe("f647d34b5793cea752bc5b892d2099c92f1ced5f13b8a4ec3e4826d9f9cd0163"); expect(data.projectName).toBe("zombo.com"); diff --git a/packages/publisher/src/generate-packages.ts b/packages/publisher/src/generate-packages.ts index 0822473b91..901e30c2ef 100644 --- a/packages/publisher/src/generate-packages.ts +++ b/packages/publisher/src/generate-packages.ts @@ -76,7 +76,7 @@ async function generateTypingPackage(typing: TypingsData, version: string, dt: F await writeCommonOutputs(typing, createPackageJSON(typing, version), createReadme(typing, packageFS)); await Promise.all( - typing.files.map(async (file) => writeFile(await outputFilePath(typing, file), packageFS.readFile(file))) + typing.getFiles().map(async (file) => writeFile(await outputFilePath(typing, file), packageFS.readFile(file))) ); } @@ -179,10 +179,11 @@ export function createReadme(typing: TypingsData, packageFS: FS): string { lines.push("# Details"); lines.push(`Files were exported from ${definitelyTypedURL}/tree/${sourceBranch}/types/${typing.subDirectoryPath}.`); - if (typing.dtsFiles.length === 1 && packageFS.readFile(typing.dtsFiles[0]).length < 2500) { - const dts = typing.dtsFiles[0]; + const dtsFiles = typing.getDtsFiles(); + if (dtsFiles.length === 1 && packageFS.readFile(dtsFiles[0]).length < 2500) { + const dts = dtsFiles[0]; const url = `${definitelyTypedURL}/tree/${sourceBranch}/types/${typing.subDirectoryPath}/${dts}`; - lines.push(`## [${typing.dtsFiles[0]}](${url})`); + lines.push(`## [${dtsFiles[0]}](${url})`); lines.push("````ts"); lines.push(packageFS.readFile(dts)); lines.push("````"); diff --git a/packages/publisher/test/generate-packages.test.ts b/packages/publisher/test/generate-packages.test.ts index feaa8ba225..d991c607d9 100644 --- a/packages/publisher/test/generate-packages.test.ts +++ b/packages/publisher/test/generate-packages.test.ts @@ -96,10 +96,19 @@ testo({ readmeContainsManyDTSFilesDoesNotAmendREADME() { const rawPkg = createRawPackage(License.Apache20); const dt = defaultFS(); - dt.pkgDir("jquery").set("other.d.ts", ""); + dt.pkgDir("jquery") + .set("other.d.ts", "") + .set("OTHER_FILES.txt", "other.d.ts"); const typing = new TypingsData(dt.fs, rawPkg, /*isLatest*/ true); expect(createReadme(typing, dt.fs)).not.toContain("type T = import"); }, + generatingPackageJsonFailsWhenFilesHaveErrors() { + const rawPkg = createRawPackage(License.Apache20); + const dt = defaultFS(); + dt.pkgDir("jquery").set("unused.d.ts", ""); + const typing = new TypingsData(dt.fs, rawPkg, /*isLatest*/ true); + expect(() => createPackageJSON(typing, "1.0")).toThrowError("Unused file"); + }, basicPackageJson() { const typing = new TypingsData(defaultFS().fs, createRawPackage(License.MIT), /*isLatest*/ true); expect(createPackageJSON(typing, "1.0")).toEqual(`{ From 8e194e2475bfe25454b8ed95219a5f12e150daa3 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 18 Oct 2023 17:06:17 -0700 Subject: [PATCH 05/12] Throw if any packages had errors during publishing --- packages/definitions-parser/src/packages.ts | 13 +++++++------ packages/dtslint-runner/src/check-parse-results.ts | 2 +- packages/publisher/src/calculate-versions.ts | 3 +++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index 15dc4398a5..b72ba43695 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -39,7 +39,8 @@ export class AllPackages { return new TypingsData(dt, versions[0], /*isLatest*/ true); } - private readonly errors: Map = new Map(); + /** Keys are `typesDirectoryName` strings */ + readonly errors: Map = new Map(); private isComplete = false; private moduleResolutionHost = createModuleResolutionHost(this.dt, this.dt.debugPath()) @@ -58,7 +59,7 @@ export class AllPackages { return await this.tryGetTypingsData(dep) !== undefined; } - getErrors() { + getErrorsAsArray() { return Array.from(this.errors.entries()).map(([name, errors]) => `${name}: ${errors.join("\n")}`); } @@ -182,12 +183,12 @@ export class AllPackages { return; } const types = this.dt.subDir("types"); - for (const typesDirectoryName of types.readdir()) { + await Promise.all(types.readdir().map(async (typesDirectoryName) => { if (!types.isDirectory(typesDirectoryName) || !types.subDir(typesDirectoryName).exists("package.json")) { - continue; + return; } - this.tryGetTypingsVersions(typesDirectoryName); - } + await this.tryGetTypingsVersions(typesDirectoryName); + })); this.isComplete = true; } } diff --git a/packages/dtslint-runner/src/check-parse-results.ts b/packages/dtslint-runner/src/check-parse-results.ts index e58c773e68..08c39e1c79 100644 --- a/packages/dtslint-runner/src/check-parse-results.ts +++ b/packages/dtslint-runner/src/check-parse-results.ts @@ -27,5 +27,5 @@ export async function checkParseResults(allPackages: AllPackages): Promise Date: Wed, 18 Oct 2023 17:06:43 -0700 Subject: [PATCH 06/12] Format --- .../src/lib/definition-parser.ts | 28 +++---- packages/definitions-parser/src/packages.ts | 77 +++++++++++++------ packages/definitions-parser/test/git.test.ts | 5 +- .../definitions-parser/test/packages.test.ts | 13 ++-- packages/publisher/src/calculate-versions.ts | 4 +- packages/publisher/src/lib/versions.ts | 12 +-- .../publisher/test/generate-packages.test.ts | 6 +- packages/retag/src/index.ts | 9 +-- 8 files changed, 87 insertions(+), 67 deletions(-) diff --git a/packages/definitions-parser/src/lib/definition-parser.ts b/packages/definitions-parser/src/lib/definition-parser.ts index 135ff96c94..709eb22dd6 100644 --- a/packages/definitions-parser/src/lib/definition-parser.ts +++ b/packages/definitions-parser/src/lib/definition-parser.ts @@ -8,16 +8,7 @@ import { validatePackageJson, } from "@definitelytyped/header-parser"; import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; -import { - FS, - assertDefined, - filter, - hasWindowsSlashes, - join, - sort, - split, - withoutStart -} from "@definitelytyped/utils"; +import { FS, assertDefined, filter, hasWindowsSlashes, join, sort, split, withoutStart } from "@definitelytyped/utils"; import assert from "assert"; import path from "path"; import * as ts from "typescript"; @@ -50,7 +41,10 @@ function formattedLibraryVersion(typingsDataRaw: TypingsDataRaw): `${number}.${n return `${typingsDataRaw.header.libraryMajorVersion}.${typingsDataRaw.header.libraryMinorVersion}`; } -export async function getTypingInfo(packageNameOrTypesDirectoryName: string, dt: FS): Promise { +export async function getTypingInfo( + packageNameOrTypesDirectoryName: string, + dt: FS +): Promise { const errors = []; if (packageNameOrTypesDirectoryName !== packageNameOrTypesDirectoryName.toLowerCase()) { errors.push(`Package name \`${packageNameOrTypesDirectoryName}\` should be strictly lowercase`); @@ -95,11 +89,7 @@ export async function getTypingInfo(packageNameOrTypesDirectoryName: string, dt: // tslint:disable-next-line:non-literal-fs-path -- Not a reference to the fs package const ls = fs.readdir(directoryName); - const result = await getPackageJsonInfoForPackage( - packageNameOrTypesDirectoryName, - ls, - fs.subDir(directoryName), - ); + const result = await getPackageJsonInfoForPackage(packageNameOrTypesDirectoryName, ls, fs.subDir(directoryName)); if (Array.isArray(result)) { errors.push(...result); return result; @@ -190,7 +180,7 @@ export function parseVersionFromDirectoryName( async function getPackageJsonInfoForPackage( typingsPackageName: string, ls: readonly string[], - fs: FS, + fs: FS ): Promise | string[]> { const errors = []; const typesVersionAndPackageJson = getTypesVersionsAndPackageJson(ls); @@ -208,7 +198,7 @@ async function getPackageJsonInfoForPackage( readonly exports?: unknown; readonly type?: unknown; }; - + const packageJsonType = checkPackageJsonType(packageJson.type, packageJsonName); if (Array.isArray(packageJsonType)) { errors.push(...packageJsonType); @@ -403,7 +393,7 @@ function getFilesForSingleTypeScriptVersion( const declFiles = sort(types.keys()); return { typescriptVersion, - declFiles: typescriptVersion === undefined ? declFiles : declFiles.map(f => `ts${typescriptVersion}/${f}`), + declFiles: typescriptVersion === undefined ? declFiles : declFiles.map((f) => `ts${typescriptVersion}/${f}`), tsconfigPathsForHash: JSON.stringify(tsconfig.compilerOptions?.paths), }; } diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index b72ba43695..0809231da5 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -1,9 +1,25 @@ import assert = require("assert"); import { Header, License, Owner } from "@definitelytyped/header-parser"; import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; -import { Dir, FS, InMemoryFS, assertDefined, computeHash, createModuleResolutionHost, mapDefined, unique, unmangleScopedPackage } from "@definitelytyped/utils"; +import { + Dir, + FS, + InMemoryFS, + assertDefined, + computeHash, + createModuleResolutionHost, + mapDefined, + unique, + unmangleScopedPackage, +} from "@definitelytyped/utils"; import * as semver from "semver"; -import { FilesForSingleTypeScriptVersion, getFiles, getTypingInfo, parseVersionFromDirectoryName, readFileAndThrowOnBOM } from "./lib/definition-parser"; +import { + FilesForSingleTypeScriptVersion, + getFiles, + getTypingInfo, + parseVersionFromDirectoryName, + readFileAndThrowOnBOM, +} from "./lib/definition-parser"; import { scopeName, typesDirectoryName } from "./lib/settings"; import { slicePrefixes } from "./lib/utils"; @@ -19,8 +35,8 @@ export class AllPackages { return new AllPackages( fs, new Map(Object.entries(typingsVersionsRaw).map(([name, raw]) => [name, new TypingsVersions(fs, raw)])), - notNeeded, - ) + notNeeded + ); } /** Use for `--single` tasks only. Do *not* call this in a loop! */ @@ -42,13 +58,13 @@ export class AllPackages { /** Keys are `typesDirectoryName` strings */ readonly errors: Map = new Map(); private isComplete = false; - private moduleResolutionHost = createModuleResolutionHost(this.dt, this.dt.debugPath()) + private moduleResolutionHost = createModuleResolutionHost(this.dt, this.dt.debugPath()); private constructor( private dt: FS, /** Keys are `typesDirectoryName` strings */ private readonly types: Map, - private readonly notNeeded: readonly NotNeededPackage[], + private readonly notNeeded: readonly NotNeededPackage[] ) {} getNotNeededPackage(typesDirectoryName: string): NotNeededPackage | undefined { @@ -56,9 +72,9 @@ export class AllPackages { } async hasTypingFor(dep: PackageId): Promise { - return await this.tryGetTypingsData(dep) !== undefined; + return (await this.tryGetTypingsData(dep)) !== undefined; } - + getErrorsAsArray() { return Array.from(this.errors.entries()).map(([name, errors]) => `${name}: ${errors.join("\n")}`); } @@ -123,7 +139,7 @@ export class AllPackages { versions && versions.tryGet(new semver.Range(pkg.version === "*" ? "*" : `^${formatTypingVersion(pkg.version)}`)) ); } - + async tryGetTypingsVersions(typesDirectoryName: string): Promise { let versions = this.types.get(typesDirectoryName); if (versions) { @@ -145,9 +161,8 @@ export class AllPackages { return versions; } - async allPackages(): Promise { - return [...await this.allTypings(), ...this.allNotNeeded()]; + return [...(await this.allTypings()), ...this.allNotNeeded()]; } /** Note: this includes older version directories (`foo/v0`) */ @@ -177,18 +192,20 @@ export class AllPackages { } } } - + private async readAllTypings() { if (this.isComplete) { return; } const types = this.dt.subDir("types"); - await Promise.all(types.readdir().map(async (typesDirectoryName) => { - if (!types.isDirectory(typesDirectoryName) || !types.subDir(typesDirectoryName).exists("package.json")) { - return; - } - await this.tryGetTypingsVersions(typesDirectoryName); - })); + await Promise.all( + types.readdir().map(async (typesDirectoryName) => { + if (!types.isDirectory(typesDirectoryName) || !types.subDir(typesDirectoryName).exists("package.json")) { + return; + } + await this.tryGetTypingsVersions(typesDirectoryName); + }) + ); this.isComplete = true; } } @@ -404,7 +421,11 @@ export class TypingsVersions { */ private readonly versions: semver.SemVer[]; - constructor(dt: FS, data: TypingsVersionsRaw, private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath())) { + constructor( + dt: FS, + data: TypingsVersionsRaw, + private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath()) + ) { /** * Sorted from latest to oldest so that we publish the current version first. * This is important because older versions repeatedly reset the "latest" tag to the current version. @@ -413,7 +434,10 @@ export class TypingsVersions { this.versions.sort(semver.rcompare); this.map = new Map( - this.versions.map((version, i) => [version, new TypingsData(dt, data[`${version.major}.${version.minor}`], !i, this.moduleResolutionHost)]) + this.versions.map((version, i) => [ + version, + new TypingsData(dt, data[`${version.major}.${version.minor}`], !i, this.moduleResolutionHost), + ]) ); } @@ -443,7 +467,12 @@ export class TypingsVersions { } export class TypingsData extends PackageBase { - constructor(private dt: FS, private readonly data: TypingsDataRaw, readonly isLatest: boolean, private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath())) { + constructor( + private dt: FS, + private readonly data: TypingsDataRaw, + readonly isLatest: boolean, + private moduleResolutionHost = createModuleResolutionHost(dt, dt.debugPath()) + ) { super(); } @@ -518,11 +547,11 @@ export class TypingsData extends PackageBase { private _contentHash: string | undefined; get contentHash(): string { - return this._contentHash ??= hash( + return (this._contentHash ??= hash( [...this.getFiles(), "package.json"], mapDefined(this.typesVersionsFiles!, (a) => a.tsconfigPathsForHash), this.dt.subDir("types").subDir(this.typesDirectoryName) - ); + )); } get projectName(): string | undefined { return this.data.header.projects[0]; @@ -614,4 +643,4 @@ function hash(files: readonly string[], tsconfigPathsForHash: readonly string[], allContent += path; } return computeHash(allContent); -} \ No newline at end of file +} diff --git a/packages/definitions-parser/test/git.test.ts b/packages/definitions-parser/test/git.test.ts index 740f16581b..994b5939c6 100644 --- a/packages/definitions-parser/test/git.test.ts +++ b/packages/definitions-parser/test/git.test.ts @@ -44,7 +44,10 @@ When removing packages, you should only delete files that are a part of removed }, async deleteInOtherPackage() { expect( - await getNotNeededPackages(allPackages, [...deleteJestDiffs, { status: "D", file: "types/most-recent/extra-tests.ts" }]) + await getNotNeededPackages(allPackages, [ + ...deleteJestDiffs, + { status: "D", file: "types/most-recent/extra-tests.ts" }, + ]) ).toEqual({ ok: jestNotNeeded }); }, async extraneousFile() { diff --git a/packages/definitions-parser/test/packages.test.ts b/packages/definitions-parser/test/packages.test.ts index c8adb499b5..45928993cd 100644 --- a/packages/definitions-parser/test/packages.test.ts +++ b/packages/definitions-parser/test/packages.test.ts @@ -25,9 +25,7 @@ describe(AllPackages, () => { it("applies path mappings to test dependencies", async () => { const pkg = await allPackages.tryGetLatestVersion("has-older-test-dependency"); for await (const { id } of allPackages.allDependencyTypings(pkg!)) { - expect(id).toEqual( - { typesDirectoryName: "jquery", version: { major: 1, minor: 0 } }, - ); + expect(id).toEqual({ typesDirectoryName: "jquery", version: { major: 1, minor: 0 } }); } }); @@ -121,9 +119,12 @@ describe(TypingsData, () => { beforeEach(() => { const dt = createMockDT(); dt.pkgDir("known") - .set("package.json", JSON.stringify({ - name: "@types/known" - })) + .set( + "package.json", + JSON.stringify({ + name: "@types/known", + }) + ) .set("index.d.ts", "declare const x: number;") .set("tsconfig.json", `{ "files": ["index.d.ts"] }`); diff --git a/packages/publisher/src/calculate-versions.ts b/packages/publisher/src/calculate-versions.ts index 44bdf5a426..79feb3d266 100644 --- a/packages/publisher/src/calculate-versions.ts +++ b/packages/publisher/src/calculate-versions.ts @@ -91,7 +91,9 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE return undefined; }); if (allPackages.errors.size) { - throw new Error(`Cannot determine if packages with errors need to be published:\n\n${allPackages.getErrorsAsArray().join("\n")}`); + throw new Error( + `Cannot determine if packages with errors need to be published:\n\n${allPackages.getErrorsAsArray().join("\n")}` + ); } return { changedTypings: compact(changedTypings), changedNotNeededPackages: compact(changedNotNeededPackages) }; } diff --git a/packages/publisher/src/lib/versions.ts b/packages/publisher/src/lib/versions.ts index db88919c0a..7a000fb751 100644 --- a/packages/publisher/src/lib/versions.ts +++ b/packages/publisher/src/lib/versions.ts @@ -37,11 +37,13 @@ export interface ChangedPackages { export async function readChangedPackages(allPackages: AllPackages): Promise { const json = (await readDataFile("calculate-versions", versionsFilename)) as ChangedPackagesJson; return { - changedTypings: await Promise.all(json.changedTypings.map(async ({ id, version, latestVersion }) => ({ - pkg: await allPackages.getTypingsData(id), - version, - latestVersion, - }))), + changedTypings: await Promise.all( + json.changedTypings.map(async ({ id, version, latestVersion }) => ({ + pkg: await allPackages.getTypingsData(id), + version, + latestVersion, + })) + ), changedNotNeededPackages: json.changedNotNeededPackages.map((id) => assertDefined(allPackages.getNotNeededPackage(id)) ), diff --git a/packages/publisher/test/generate-packages.test.ts b/packages/publisher/test/generate-packages.test.ts index d991c607d9..38c21f8e29 100644 --- a/packages/publisher/test/generate-packages.test.ts +++ b/packages/publisher/test/generate-packages.test.ts @@ -38,7 +38,7 @@ function defaultFS() { dt.pkgDir("jquery") .set("package.json", JSON.stringify({ name: "@types/jquery" })) .set("tsconfig.json", `{ "files": ["index.d.ts", "jquery-tests.ts"] }`) - .set("index.d.ts",`type T = import("./types");\n`) + .set("index.d.ts", `type T = import("./types");\n`) .set("jquery-tests.ts", "// tests"); return dt; } @@ -96,9 +96,7 @@ testo({ readmeContainsManyDTSFilesDoesNotAmendREADME() { const rawPkg = createRawPackage(License.Apache20); const dt = defaultFS(); - dt.pkgDir("jquery") - .set("other.d.ts", "") - .set("OTHER_FILES.txt", "other.d.ts"); + dt.pkgDir("jquery").set("other.d.ts", "").set("OTHER_FILES.txt", "other.d.ts"); const typing = new TypingsData(dt.fs, rawPkg, /*isLatest*/ true); expect(createReadme(typing, dt.fs)).not.toContain("type T = import"); }, diff --git a/packages/retag/src/index.ts b/packages/retag/src/index.ts index 6fda41bb25..4133d8f3e9 100644 --- a/packages/retag/src/index.ts +++ b/packages/retag/src/index.ts @@ -16,12 +16,7 @@ import { cacheDir, nAtATime, } from "@definitelytyped/utils"; -import { - AnyPackage, - TypingsData, - AllPackages, - getDefinitelyTyped, -} from "@definitelytyped/definitions-parser"; +import { AnyPackage, TypingsData, AllPackages, getDefinitelyTyped } from "@definitelytyped/definitions-parser"; import * as pacote from "pacote"; import * as semver from "semver"; @@ -54,7 +49,7 @@ async function tag(dry: boolean, _nProcesses: number, name?: string) { const publishClient = await NpmPublishClient.create(token, {}); if (name) { - const pkg = await AllPackages.readSingle(dt,name); + const pkg = await AllPackages.readSingle(dt, name); const version = await getLatestTypingVersion(pkg); await updateTypeScriptVersionTags(pkg, version, publishClient, consoleLogger.info, dry); await updateLatestTag(pkg.name, version, publishClient, consoleLogger.info, dry); From ab776400e5a0e396ca6fce92033f32ea742a93b9 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 19 Oct 2023 09:11:58 -0700 Subject: [PATCH 07/12] Warm the allowedPackageJsonDependencies cache --- packages/definitions-parser/src/packages.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index 0809231da5..3dc7b1cf7d 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -20,7 +20,7 @@ import { parseVersionFromDirectoryName, readFileAndThrowOnBOM, } from "./lib/definition-parser"; -import { scopeName, typesDirectoryName } from "./lib/settings"; +import { getAllowedPackageJsonDependencies, scopeName, typesDirectoryName } from "./lib/settings"; import { slicePrefixes } from "./lib/utils"; export class AllPackages { @@ -197,6 +197,8 @@ export class AllPackages { if (this.isComplete) { return; } + // populate cache so every directory doesn't try to request this from GH + await getAllowedPackageJsonDependencies(); const types = this.dt.subDir("types"); await Promise.all( types.readdir().map(async (typesDirectoryName) => { From 72d0a5749234d22173ace9227bba7a2fb5eab361 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 19 Oct 2023 09:28:32 -0700 Subject: [PATCH 08/12] Remove unused options/types/utils --- .../src/get-definitely-typed.ts | 2 - .../test/get-affected-packages.test.ts | 2 +- .../test/get-definitely-typed.test.ts | 1 - .../dtslint-runner/src/check-parse-results.ts | 2 +- packages/dtslint-runner/src/main.ts | 4 +- .../src/prepareAffectedPackages.ts | 2 - .../dtslint-runner/src/prepareAllPackages.ts | 2 - packages/publisher/src/generate-packages.ts | 2 +- packages/publisher/src/lib/common.ts | 2 - packages/publisher/src/main.ts | 1 - packages/publisher/src/publish-packages.ts | 2 +- packages/retag/src/index.ts | 2 +- packages/utils/src/process.ts | 67 ------------------- 13 files changed, 7 insertions(+), 84 deletions(-) diff --git a/packages/definitions-parser/src/get-definitely-typed.ts b/packages/definitions-parser/src/get-definitely-typed.ts index 485a828c99..3195b09ec6 100644 --- a/packages/definitions-parser/src/get-definitely-typed.ts +++ b/packages/definitions-parser/src/get-definitely-typed.ts @@ -15,8 +15,6 @@ export interface ParseDefinitionsOptions { readonly definitelyTypedPath: string | undefined; /** Whether to show progress bars. Good when running locally, bad when running in CI. */ readonly progress: boolean; - /** Disabled in CI since it has problems logging errors from other processes. */ - readonly parseInParallel: boolean; } export async function getDefinitelyTyped(options: ParseDefinitionsOptions, log: LoggerWithErrors): Promise { diff --git a/packages/definitions-parser/test/get-affected-packages.test.ts b/packages/definitions-parser/test/get-affected-packages.test.ts index ad036ce600..410fd659c3 100644 --- a/packages/definitions-parser/test/get-affected-packages.test.ts +++ b/packages/definitions-parser/test/get-affected-packages.test.ts @@ -73,7 +73,7 @@ testo({ expect(packageNames).toEqual(new Set(["mistake"])); expect(dependents).toEqual(new Set([])); }, - olderVersion() { + async olderVersion() { const packageOutput = `/dt/types/jquery`; const dependentOutput = `/dt/types/jquery /dt/types/has-older-test-dependency diff --git a/packages/definitions-parser/test/get-definitely-typed.test.ts b/packages/definitions-parser/test/get-definitely-typed.test.ts index 032b1d235f..15a1806359 100644 --- a/packages/definitions-parser/test/get-definitely-typed.test.ts +++ b/packages/definitions-parser/test/get-definitely-typed.test.ts @@ -7,7 +7,6 @@ testo({ const dt = await getDefinitelyTyped( { definitelyTypedPath: undefined, - parseInParallel: false, progress: false, }, quietLoggerWithErrors()[0] diff --git a/packages/dtslint-runner/src/check-parse-results.ts b/packages/dtslint-runner/src/check-parse-results.ts index 08c39e1c79..2eb88e699a 100644 --- a/packages/dtslint-runner/src/check-parse-results.ts +++ b/packages/dtslint-runner/src/check-parse-results.ts @@ -3,7 +3,7 @@ import { AllPackages, getDefinitelyTyped } from "@definitelytyped/definitions-pa import { assertDefined } from "@definitelytyped/utils"; import * as semver from "semver"; if (require.main === module) { - const options = { definitelyTypedPath: undefined, progress: false, parseInParallel: false }; + const options = { definitelyTypedPath: undefined, progress: false }; getDefinitelyTyped(options, console).then((dt) => { return checkParseResults(AllPackages.fromFS(dt)); }); diff --git a/packages/dtslint-runner/src/main.ts b/packages/dtslint-runner/src/main.ts index 9d10d26f24..3f9dd76e62 100644 --- a/packages/dtslint-runner/src/main.ts +++ b/packages/dtslint-runner/src/main.ts @@ -48,8 +48,8 @@ export async function runDTSLint({ const typesPath = joinPaths(definitelyTypedPath, "types"); const { packageNames, dependents } = onlyRunAffectedPackages - ? await prepareAffectedPackages(definitelyTypedPath, nProcesses) - : await prepareAllPackages(definitelyTypedPath, definitelyTypedAcquisition.kind === "clone", nProcesses); + ? await prepareAffectedPackages(definitelyTypedPath) + : await prepareAllPackages(definitelyTypedPath, definitelyTypedAcquisition.kind === "clone"); if (!noInstall && !localTypeScriptPath) { if (onlyTestTsNext) { diff --git a/packages/dtslint-runner/src/prepareAffectedPackages.ts b/packages/dtslint-runner/src/prepareAffectedPackages.ts index 59925a53e4..0f44eade66 100644 --- a/packages/dtslint-runner/src/prepareAffectedPackages.ts +++ b/packages/dtslint-runner/src/prepareAffectedPackages.ts @@ -9,13 +9,11 @@ import { checkParseResults } from "./check-parse-results"; export async function prepareAffectedPackages( definitelyTypedPath: string, - nProcesses: number ): Promise { const log = loggerWithErrors()[0]; const options = { definitelyTypedPath, progress: false, - parseInParallel: nProcesses > 1, }; const dt = await getDefinitelyTyped(options, log); const allPackages = AllPackages.fromFS(dt); diff --git a/packages/dtslint-runner/src/prepareAllPackages.ts b/packages/dtslint-runner/src/prepareAllPackages.ts index 80d9ad1a54..10d4d61ece 100644 --- a/packages/dtslint-runner/src/prepareAllPackages.ts +++ b/packages/dtslint-runner/src/prepareAllPackages.ts @@ -5,13 +5,11 @@ import { checkParseResults } from "./check-parse-results"; export async function prepareAllPackages( definitelyTypedPath: string, clone: boolean, - nProcesses: number ): Promise { const [log] = loggerWithErrors(); const options = { definitelyTypedPath, progress: false, - parseInParallel: nProcesses > 1, }; const dt = await getDefinitelyTyped(options, log); const allPackages = AllPackages.fromFS(dt); diff --git a/packages/publisher/src/generate-packages.ts b/packages/publisher/src/generate-packages.ts index 60d54d714c..098bae3796 100644 --- a/packages/publisher/src/generate-packages.ts +++ b/packages/publisher/src/generate-packages.ts @@ -36,7 +36,7 @@ if (require.main === module) { const tgz = !!yargs.argv.tgz; logUncaughtErrors(async () => { const log = loggerWithErrors()[0]; - const options = { ...defaultLocalOptions, definitelyTypedPath: outputDirPath, parseInParallel: true }; + const options = { ...defaultLocalOptions, definitelyTypedPath: outputDirPath }; if (yargs.argv.path) { options.definitelyTypedPath = yargs.argv.path as string; } diff --git a/packages/publisher/src/lib/common.ts b/packages/publisher/src/lib/common.ts index f886282d04..ab44960cf1 100644 --- a/packages/publisher/src/lib/common.ts +++ b/packages/publisher/src/lib/common.ts @@ -14,11 +14,9 @@ export interface TesterOptions extends ParseDefinitionsOptions { export const defaultLocalOptions: TesterOptions = { definitelyTypedPath: "../../../DefinitelyTyped", progress: true, - parseInParallel: !process.env.VSCODE_INSPECTOR_OPTIONS, }; export const defaultRemoteOptions: ParseDefinitionsOptions = { definitelyTypedPath: undefined, progress: false, - parseInParallel: false, }; diff --git a/packages/publisher/src/main.ts b/packages/publisher/src/main.ts index c1cd8ddd2f..6e3567a40b 100644 --- a/packages/publisher/src/main.ts +++ b/packages/publisher/src/main.ts @@ -25,7 +25,6 @@ export default async function main() { fetcher, { definitelyTypedPath, - parseInParallel: false, progress: false, }, log diff --git a/packages/publisher/src/publish-packages.ts b/packages/publisher/src/publish-packages.ts index 8f8e9336b6..e6c90d5e8d 100644 --- a/packages/publisher/src/publish-packages.ts +++ b/packages/publisher/src/publish-packages.ts @@ -18,7 +18,7 @@ import { getSecret, Secret } from "./lib/secrets"; if (require.main === module) { const dry = !!yargs.argv.dry; logUncaughtErrors(async () => { - const options = { ...defaultLocalOptions, parseInParallel: true }; + const options = { ...defaultLocalOptions }; if (yargs.argv.path) { options.definitelyTypedPath = yargs.argv.path as string; } diff --git a/packages/retag/src/index.ts b/packages/retag/src/index.ts index 4133d8f3e9..c521ea6c41 100644 --- a/packages/retag/src/index.ts +++ b/packages/retag/src/index.ts @@ -43,7 +43,7 @@ async function main() { */ async function tag(dry: boolean, _nProcesses: number, name?: string) { const log = loggerWithErrors()[0]; - const options = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true }; + const options = { definitelyTypedPath: "../DefinitelyTyped", progress: true }; const dt = await getDefinitelyTyped(options, log); const token = process.env.NPM_TOKEN as string; diff --git a/packages/utils/src/process.ts b/packages/utils/src/process.ts index c925dc0aa0..fed50843fd 100644 --- a/packages/utils/src/process.ts +++ b/packages/utils/src/process.ts @@ -27,73 +27,6 @@ export async function execAndThrowErrors(cmd: string, cwd?: string): Promise { - readonly inputs: readonly In[]; - readonly commandLineArgs: string[]; - readonly workerFile: string; - readonly nProcesses: number; - handleOutput(output: unknown): void; -} -export function runWithChildProcesses({ - inputs, - commandLineArgs, - workerFile, - nProcesses, - handleOutput, -}: RunWithChildProcessesOptions): Promise { - return new Promise(async (resolve, reject) => { - const nPerProcess = Math.floor(inputs.length / nProcesses); - let processesLeft = nProcesses; - let rejected = false; - const allChildren: ChildProcess[] = []; - for (let i = 0; i < nProcesses; i++) { - const lo = nPerProcess * i; - const hi = i === nProcesses - 1 ? inputs.length : lo + nPerProcess; - let outputsLeft = hi - lo; // Expect one output per input - if (outputsLeft === 0) { - // No work for this process to do, so don't launch it - processesLeft--; - continue; - } - const child = fork(workerFile, commandLineArgs, { - execArgv: await getChildProcessExecArgv(i), - }); - allChildren.push(child); - child.send(inputs.slice(lo, hi)); - child.on("message", (outputMessage) => { - handleOutput(outputMessage as unknown); - assert(outputsLeft > 0); - outputsLeft--; - if (outputsLeft === 0) { - assert(processesLeft > 0); - processesLeft--; - if (processesLeft === 0) { - resolve(); - } - child.kill(); - } - }); - child.on("disconnect", () => { - if (outputsLeft !== 0) { - fail(new Error(`disconnect with ${outputsLeft} outputs left`)); - } - }); - child.on("close", () => { - assert(rejected || outputsLeft === 0); - }); - child.on("error", fail); - } - - function fail(e: Error): void { - rejected = true; - for (const child of allChildren) { - child.kill(); - } - reject(e); - } - }); -} - export const enum CrashRecoveryState { Normal, Retry, From 35a883d22cef5cd21df1bcec8cfec5210bc7386d Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 19 Oct 2023 10:10:44 -0700 Subject: [PATCH 09/12] Add changeset --- .changeset/slimy-pumpkins-camp.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changeset/slimy-pumpkins-camp.md diff --git a/.changeset/slimy-pumpkins-camp.md b/.changeset/slimy-pumpkins-camp.md new file mode 100644 index 0000000000..c6a112f88d --- /dev/null +++ b/.changeset/slimy-pumpkins-camp.md @@ -0,0 +1,9 @@ +--- +"@definitelytyped/definitions-parser": patch +"@definitelytyped/dtslint-runner": patch +"@definitelytyped/publisher": patch +"@definitelytyped/retag": patch +"@definitelytyped/utils": patch +--- + +Make AllPackages lazy and asynchronous From 1a042e77b0f440537d991bff3127d959d22f226e Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 19 Oct 2023 11:01:40 -0700 Subject: [PATCH 10/12] Clean up --- packages/definitions-parser/src/lib/definition-parser.ts | 7 ++----- packages/retag/src/index.ts | 8 +++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/definitions-parser/src/lib/definition-parser.ts b/packages/definitions-parser/src/lib/definition-parser.ts index 709eb22dd6..8228b62226 100644 --- a/packages/definitions-parser/src/lib/definition-parser.ts +++ b/packages/definitions-parser/src/lib/definition-parser.ts @@ -292,11 +292,6 @@ export function getFiles( } return [dataForRoot, ...dataForOtherTypesVersions] as FilesForSingleTypeScriptVersion[]; - // return Array.from( - // flatMap(allTypesVersions, ({ typescriptVersion, declFiles }) => - // declFiles.map((file) => (typescriptVersion === undefined ? file : `ts${typescriptVersion}/${file}`)) - // ) - // ); } export interface FilesForSingleTypeScriptVersion { @@ -345,6 +340,8 @@ function getFilesForSingleTypeScriptVersion( } catch (err) { if (err instanceof Error) { errors.push(err.message); + } else { + throw err; } return errors; } diff --git a/packages/retag/src/index.ts b/packages/retag/src/index.ts index c521ea6c41..69427c79de 100644 --- a/packages/retag/src/index.ts +++ b/packages/retag/src/index.ts @@ -3,7 +3,6 @@ import assert = require("assert"); import yargs from "yargs"; import process = require("process"); -import os = require("os"); import { TypeScriptVersion } from "@definitelytyped/typescript-versions"; import { @@ -25,12 +24,11 @@ if (require.main === module) { } async function main() { - const { dry, nProcesses, name } = yargs.options({ + const { dry, name } = yargs.options({ dry: { type: "boolean", default: false }, - nProcesses: { type: "number", default: os.cpus().length }, name: { type: "string" }, }).argv; - await tag(dry, nProcesses, name); + await tag(dry, name); } /** @@ -41,7 +39,7 @@ async function main() { * This shouldn't normally need to run, since we run `tagSingle` whenever we publish a package. * But this should be run if the way we calculate tags changes (e.g. when a new release is allowed to be tagged "latest"). */ -async function tag(dry: boolean, _nProcesses: number, name?: string) { +async function tag(dry: boolean, name?: string) { const log = loggerWithErrors()[0]; const options = { definitelyTypedPath: "../DefinitelyTyped", progress: true }; const dt = await getDefinitelyTyped(options, log); From 2dd1f68757221a54335d872ada27869ad18e4719 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 19 Oct 2023 11:52:20 -0700 Subject: [PATCH 11/12] Simplify getFiles error handling --- .../definitions-parser/src/lib/definition-parser.ts | 4 ++-- packages/definitions-parser/src/packages.ts | 13 +------------ .../test/definition-parser.test.ts | 7 +------ 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/packages/definitions-parser/src/lib/definition-parser.ts b/packages/definitions-parser/src/lib/definition-parser.ts index 8228b62226..eb54d89a7a 100644 --- a/packages/definitions-parser/src/lib/definition-parser.ts +++ b/packages/definitions-parser/src/lib/definition-parser.ts @@ -252,7 +252,7 @@ export function getFiles( dt: FS, typingsData: TypingsData, moduleResolutionHost: ts.ModuleResolutionHost -): readonly FilesForSingleTypeScriptVersion[] | { errors: string[] } { +): readonly FilesForSingleTypeScriptVersion[] { const errors = []; const rootDir = dt.subDir("types").subDir(typingsData.typesDirectoryName); const typesVersionAndPackageJson = getTypesVersionsAndPackageJson(rootDir.readdir()); @@ -288,7 +288,7 @@ export function getFiles( }); if (errors.length) { - return { errors }; + throw new Error(`Errors encountered resolving files for ${typingsData.name}:\n${errors.join("\n")}`); } return [dataForRoot, ...dataForOtherTypesVersions] as FilesForSingleTypeScriptVersion[]; diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index a40fe07d37..7fcd7b9d2d 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -509,25 +509,14 @@ export class TypingsData extends PackageBase { } private typesVersionsFiles: readonly FilesForSingleTypeScriptVersion[] | undefined; - tryGetFiles(): readonly string[] | { errors: string[] } { + getFiles(): readonly string[] { if (!this.typesVersionsFiles) { const files = getFiles(this.dt, this, this.moduleResolutionHost); - if ("errors" in files) { - return files; - } this.typesVersionsFiles = files; } return this.typesVersionsFiles.flatMap((v) => v.declFiles); } - getFiles(): readonly string[] { - const files = this.tryGetFiles(); - if ("errors" in files) { - throw new Error(`Errors while reading package ${this.name}: ${files.errors.join("\n")}`); - } - return files; - } - getDtsFiles(): readonly string[] { return this.getFiles().filter((f) => f.endsWith(".d.ts") || f.endsWith(".d.mts") || f.endsWith(".d.cts")); } diff --git a/packages/definitions-parser/test/definition-parser.test.ts b/packages/definitions-parser/test/definition-parser.test.ts index 3867edd6b2..d6a09bba92 100644 --- a/packages/definitions-parser/test/definition-parser.test.ts +++ b/packages/definitions-parser/test/definition-parser.test.ts @@ -381,12 +381,7 @@ import route = require('@ember/routing/route'); throw new Error(raw.errors.join("\n")); } const typingData = new TypingsVersions(dt, raw).getLatest(); - const files = typingData.tryGetFiles(); - if (!("errors" in files)) { - throw new Error("Expected errors"); - } - expect(files.errors).toHaveLength(1); - expect(files.errors[0]).toMatch("Definitions must use global references to other packages"); + expect(() => typingData.getFiles()).toThrow("Definitions must use global references to other packages"); }); describe("concerning multiple versions", () => { From b0f3ea5b1df50c0625713eefab374f187fe71df6 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 19 Oct 2023 11:53:53 -0700 Subject: [PATCH 12/12] Make more stuff private --- packages/definitions-parser/src/packages.ts | 4 ++-- packages/publisher/src/calculate-versions.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/definitions-parser/src/packages.ts b/packages/definitions-parser/src/packages.ts index 7fcd7b9d2d..34b0448542 100644 --- a/packages/definitions-parser/src/packages.ts +++ b/packages/definitions-parser/src/packages.ts @@ -56,7 +56,7 @@ export class AllPackages { } /** Keys are `typesDirectoryName` strings */ - readonly errors: Map = new Map(); + private readonly errors: Map = new Map(); private isComplete = false; private moduleResolutionHost = createModuleResolutionHost(this.dt, this.dt.debugPath()); @@ -140,7 +140,7 @@ export class AllPackages { ); } - async tryGetTypingsVersions(typesDirectoryName: string): Promise { + private async tryGetTypingsVersions(typesDirectoryName: string): Promise { let versions = this.types.get(typesDirectoryName); if (versions) { return versions; diff --git a/packages/publisher/src/calculate-versions.ts b/packages/publisher/src/calculate-versions.ts index 79feb3d266..e50860a029 100644 --- a/packages/publisher/src/calculate-versions.ts +++ b/packages/publisher/src/calculate-versions.ts @@ -90,9 +90,10 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE } return undefined; }); - if (allPackages.errors.size) { + const errors = allPackages.getErrorsAsArray(); + if (errors.length) { throw new Error( - `Cannot determine if packages with errors need to be published:\n\n${allPackages.getErrorsAsArray().join("\n")}` + `Cannot determine if packages with errors need to be published:\n\n${errors.join("\n")}` ); } return { changedTypings: compact(changedTypings), changedNotNeededPackages: compact(changedNotNeededPackages) };