diff --git a/.projen/deps.json b/.projen/deps.json index 572d1c477..528cc2053 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -198,6 +198,10 @@ "name": "jsii-pacmak", "type": "build" }, + { + "name": "jsii-reflect", + "type": "build" + }, { "name": "jsii-rosetta", "type": "build" diff --git a/.projen/tasks.json b/.projen/tasks.json index 6719ab8e4..afe3adcc3 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -1214,13 +1214,13 @@ "exec": "yarn upgrade npm-check-updates" }, { - "exec": "npm-check-updates --upgrade --target=minor --peer --dep=dev --filter=@aws-cdk/aws-servicecatalogappregistry-alpha,@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@types/aws-lambda,@types/changelog-filename-regex,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/node,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,async-sema,aws-cdk-lib,aws-cdk,aws-embedded-metrics,aws-sdk,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,cdklabs-projen-project-types,changelog-filename-regex,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-node,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,feed,fs-extra,glob,got,jest,jest-junit,jsii-diff,jsii-docgen,jsii-pacmak,jsii-rosetta,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,npm-check-updates,prettier,projen,semver,spdx-license-list,standard-version,streamx,tar-stream,ts-node,typescript,uuid,yaml" + "exec": "npm-check-updates --upgrade --target=minor --peer --dep=dev --filter=@aws-cdk/aws-servicecatalogappregistry-alpha,@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@types/aws-lambda,@types/changelog-filename-regex,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/node,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,async-sema,aws-cdk-lib,aws-cdk,aws-embedded-metrics,aws-sdk,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,cdklabs-projen-project-types,changelog-filename-regex,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-node,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,feed,fs-extra,glob,got,jest,jest-junit,jsii-diff,jsii-docgen,jsii-pacmak,jsii-reflect,jsii-rosetta,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,npm-check-updates,prettier,projen,semver,spdx-license-list,standard-version,streamx,tar-stream,ts-node,typescript,uuid,yaml" }, { "exec": "yarn install --check-files" }, { - "exec": "yarn upgrade @aws-cdk/aws-servicecatalogappregistry-alpha @aws-sdk/client-sfn @jsii/spec @octokit/rest @types/aws-lambda @types/changelog-filename-regex @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk-lib aws-cdk aws-embedded-metrics aws-sdk aws-sdk-mock aws-xray-sdk-core case cdk-dia cdk-watchful cdklabs-projen-project-types changelog-filename-regex construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-node eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-rosetta JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata npm-check-updates prettier projen semver spdx-license-list standard-version streamx tar-stream ts-node typescript uuid yaml" + "exec": "yarn upgrade @aws-cdk/aws-servicecatalogappregistry-alpha @aws-sdk/client-sfn @jsii/spec @octokit/rest @types/aws-lambda @types/changelog-filename-regex @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk-lib aws-cdk aws-embedded-metrics aws-sdk aws-sdk-mock aws-xray-sdk-core case cdk-dia cdk-watchful cdklabs-projen-project-types changelog-filename-regex construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-node eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-reflect jsii-rosetta JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata npm-check-updates prettier projen semver spdx-license-list standard-version streamx tar-stream ts-node typescript uuid yaml" }, { "exec": "npx projen" diff --git a/.projenrc.ts b/.projenrc.ts index 1e9ef5878..03eb00ae8 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -59,6 +59,7 @@ const project = new CdklabsConstructLibrary({ 'fs-extra', 'got', 'JSONStream', + 'jsii-reflect', 'semver', 'spdx-license-list', 'streamx', diff --git a/package.json b/package.json index dbae8c153..3d38c1593 100644 --- a/package.json +++ b/package.json @@ -164,6 +164,7 @@ "jsii-diff": "^1.91.0", "jsii-docgen": "^9.2.2", "jsii-pacmak": "^1.91.0", + "jsii-reflect": "^1.91.0", "jsii-rosetta": "5.1.x", "JSONStream": "^1.3.5", "markdown-it": "^13.0.2", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index cb39784a6..1cd86a3fb 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -6857,7 +6857,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -19876,7 +19876,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -32551,7 +32551,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -45337,7 +45337,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -58042,7 +58042,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap b/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap index 9256a4b0c..012ea8d34 100644 --- a/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap +++ b/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap @@ -229,7 +229,7 @@ exports[`CodeArtifact repository 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -2544,7 +2544,7 @@ exports[`VPC Endpoints 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -5021,7 +5021,7 @@ exports[`VPC Endpoints and CodeArtifact repository 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -7332,7 +7332,7 @@ exports[`basic use 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts b/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts index 917811437..e81d81ea9 100644 --- a/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts +++ b/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts @@ -15,6 +15,7 @@ import { MarkdownDocument } from 'jsii-docgen/lib/docgen/render/markdown-doc'; import { MarkdownRenderer } from 'jsii-docgen/lib/docgen/render/markdown-render'; import { Documentation } from 'jsii-docgen/lib/docgen/view/documentation'; +import { Submodule } from 'jsii-reflect'; import type { TransliteratorInput } from '../../../backend/payload-schema'; import { reset } from '../../../backend/shared/aws.lambda-shared'; import * as constants from '../../../backend/shared/constants'; @@ -93,6 +94,10 @@ describe('VPC Endpoints', () => { }; } } + + public async listSubmodules(): Promise { + return []; + } } forPackage.mockImplementation(async (_: string) => { @@ -229,6 +234,9 @@ test('corrupt assembly marker is uploaded for the necessary languages', async () public async toJson() { throw new CorruptedAssemblyError(); } + public async listSubmodules(): Promise { + return []; + } } forPackage.mockImplementation(async (_: string) => { @@ -286,6 +294,9 @@ test('corrupt assembly and uninstallable markers are deleted', async () => { public async toJson() { return new MarkdownDocument(); } + public async listSubmodules(): Promise { + return []; + } } forPackage.mockImplementation(async (_: string) => { @@ -392,14 +403,29 @@ test('uploads a file per language (scoped package)', async () => { }); test('uploads a file per submodule (unscoped package)', async () => { + // GIVEN + const assembly: spec.Assembly = { + targets: { python: {} }, + submodules: { + '@scope/package-name.sub1': {}, + '@scope/package-name.sub2': {}, + }, + } as any; + + const submodules: Submodule[] = Object.keys(assembly.submodules ?? {}).map( + (sm) => ({ name: sm.substring(sm.indexOf('.') + 1) } as any) + ); + // eslint-disable-next-line @typescript-eslint/no-require-imports const forPackage = require('jsii-docgen').Documentation .forPackage as jest.MockedFunction; forPackage.mockImplementation(async (target: string) => { - return new MockDocumentation(target) as unknown as Documentation; + return new MockDocumentation( + target, + submodules + ) as unknown as Documentation; }); - // GIVEN const packageName = 'package-name'; const packageVersion = '1.2.3-dev.4'; const event: TransliteratorInput = { @@ -415,15 +441,6 @@ test('uploads a file per submodule (unscoped package)', async () => { languages: { typescript: true }, }; - const assembly: spec.Assembly = { - targets: { python: {} }, - submodules: { - '@scope/package-name.sub1': {}, - '@scope/package-name.sub2': {}, - '@scope/package-name.sub2.nested': {}, - }, - } as any; - // mock the s3ObjectExists call mockHeadRequest('package.tgz'); @@ -435,15 +452,15 @@ test('uploads a file per submodule (unscoped package)', async () => { '/docs-typescript.md', '/docs-sub1-typescript.md', '/docs-sub2-typescript.md', - '/docs-sub2.nested-typescript.md', '/docs-typescript.json', '/docs-sub1-typescript.json', - '/docs-sub2-typescript.json', - '/docs-sub2.nested-typescript.json' + '/docs-sub2-typescript.json' ); + // WHEN const { created } = await handler(event); + // THEN expect(created).toEqual([ `data/${packageName}/v${packageVersion}/docs-typescript.json`, `data/${packageName}/v${packageVersion}/docs-typescript.md`, @@ -451,8 +468,6 @@ test('uploads a file per submodule (unscoped package)', async () => { `data/${packageName}/v${packageVersion}/docs-sub1-typescript.md`, `data/${packageName}/v${packageVersion}/docs-sub2-typescript.json`, `data/${packageName}/v${packageVersion}/docs-sub2-typescript.md`, - `data/${packageName}/v${packageVersion}/docs-sub2.nested-typescript.json`, - `data/${packageName}/v${packageVersion}/docs-sub2.nested-typescript.md`, ]); }); @@ -517,6 +532,12 @@ test.each([true, false])( describe('markers for un-supported languages', () => { test('uploads ".not-supported" markers as relevant', async () => { + // GIVEN + const assembly: spec.Assembly = { + targets: { phony: {} }, + submodules: { 'package-name.sub1': {}, 'package-name.sub2': {} }, + } as any; + // eslint-disable-next-line @typescript-eslint/no-require-imports const forPackage = require('jsii-docgen').Documentation .forPackage as jest.MockedFunction; @@ -525,13 +546,17 @@ describe('markers for un-supported languages', () => { public async toJson() { throw new LanguageNotSupportedError(); } + public async listSubmodules(): Promise { + return Object.keys(assembly.submodules ?? {}).map( + (sm) => ({ name: sm.substring(sm.indexOf('.') + 1) } as any) + ); + } } forPackage.mockImplementation(async (_: string) => { return new MockDocumentation() as unknown as Documentation; }); - // GIVEN const packageName = 'package-name'; const packageVersion = '1.2.3-dev.4'; @@ -549,11 +574,6 @@ describe('markers for un-supported languages', () => { languages: { python: true }, }; - const assembly: spec.Assembly = { - targets: { phony: {} }, - submodules: { 'package-name.sub1': {}, 'package-name.sub2': {} }, - } as any; - // mock the s3ObjectExists call mockHeadRequest('package.tgz'); @@ -584,12 +604,18 @@ describe('markers for un-supported languages', () => { }); class MockDocumentation { - public constructor(private readonly target: string) {} + public constructor( + private readonly target: string, + private readonly submodules?: Submodule[] + ) {} public async toJson() { return { render: () => `{ "content": "docs for ${this.target}" }`, }; } + public async listSubmodules(): Promise { + return this.submodules ?? []; + } } function mockFetchRequests(assembly: spec.Assembly, tarball: Buffer) { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 8de1e0121..bca3ce325 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -7134,7 +7134,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:337b393c2a4f1a2f9c50e717a750c3aad8ee34bff3d8c9208d051b9b5328a06a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:10112e6c3b7556a2a3e36a0761722f6a3ecbdc18c4079a7e6cd79f0678a72845", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/backend/transliterator/transliterator.ecstask.ts b/src/backend/transliterator/transliterator.ecstask.ts index f1e897bb8..3ba91f96f 100644 --- a/src/backend/transliterator/transliterator.ecstask.ts +++ b/src/backend/transliterator/transliterator.ecstask.ts @@ -107,8 +107,7 @@ export function handler( assemblyResponse.Body.toString('utf-8') ) as Assembly; const submodules = Object.keys(assembly.submodules ?? {}).map( - // Drop the first component, which is the name of the top-level module - (s) => s.split('.').slice(1).join('.') + (s) => s.split('.')[1] ); console.log( `Assembly ${assembly.name} has ${submodules.length} submodules.` @@ -302,7 +301,10 @@ export function handler( } } await renderAndDispatch(); - for (const submodule of submodules) { + const submoduleNames = (await docs.listSubmodules()).map( + (sm) => sm.name + ); + for (const submodule of submoduleNames) { await renderAndDispatch(submodule); } }