chore: reapply refactoring for L1 resource relationships

.github/workflows/codebuild-pr-build.yml

@@ -70,7 +70,7 @@ jobs:
           sudo sysctl -w vm.max_map_count=2251954
 
       - name: Build
-        run: /bin/bash ./build.sh --ci
+        run: /bin/bash ./build.sh --ci --concurrency 10
 
       - name: Run Rosetta
         run: /bin/bash ./scripts/run-rosetta.sh

.github/workflows/pr-build.yml

@@ -65,7 +65,7 @@ jobs:
           sudo sysctl -w vm.max_map_count=2251954
 
       - name: Build
-        run: /bin/bash ./build.sh --ci --concurrency=10
+        run: /bin/bash ./build.sh --ci --concurrency 10
 
       - name: Run Rosetta
         run: /bin/bash ./scripts/run-rosetta.sh

tools/@aws-cdk/spec2cdk/lib/cdk/ast.ts

@@ -3,6 +3,7 @@ import { Module } from '@cdklabs/typewriter';
 import { AugmentationsModule } from './augmentation-generator';
 import { CannedMetricsModule } from './canned-metrics';
 import { CDK_CORE, CONSTRUCTS, ModuleImportLocations } from './cdk';
+import { SelectiveImport } from './relationship-decider';
 import { ResourceClass } from './resource-class';
 
 /**
@@ -59,7 +60,7 @@ export class AstBuilder<T extends Module> {
     for (const link of resources) {
       ast.addResource(link.entity);
     }
-
+    ast.renderImports();
     return ast;
   }
 
@@ -74,6 +75,7 @@ export class AstBuilder<T extends Module> {
 
     const ast = new AstBuilder(scope, props, aug, metrics);
     ast.addResource(resource);
+    ast.renderImports();
 
     return ast;
   }
@@ -85,6 +87,8 @@ export class AstBuilder<T extends Module> {
   public readonly resources: Record<string, string> = {};
   private nameSuffix?: string;
   private deprecated?: string;
+  public readonly selectiveImports = new Array<SelectiveImport>();
+  private readonly modulesRootLocation: string;
 
   protected constructor(
     public readonly module: T,
@@ -95,6 +99,7 @@ export class AstBuilder<T extends Module> {
     this.db = props.db;
     this.nameSuffix = props.nameSuffix;
     this.deprecated = props.deprecated;
+    this.modulesRootLocation = props.importLocations?.modulesRoot ?? '../..';
 
     CDK_CORE.import(this.module, 'cdk', { fromLocation: props.importLocations?.core });
     CONSTRUCTS.import(this.module, 'constructs');
@@ -111,6 +116,35 @@ export class AstBuilder<T extends Module> {
 
     resourceClass.build();
 
+    this.addImports(resourceClass);
     this.augmentations?.augmentResource(resource, resourceClass);
   }
+
+  private addImports(resourceClass: ResourceClass) {
+    for (const selectiveImport of resourceClass.imports) {
+      const existingModuleImport = this.selectiveImports.find(
+        (imp) => imp.moduleName === selectiveImport.moduleName,
+      );
+      if (!existingModuleImport) {
+        this.selectiveImports.push(selectiveImport);
+      } else {
+        // We need to avoid importing the same reference multiple times
+        for (const type of selectiveImport.types) {
+          if (!existingModuleImport.types.find((t) =>
+            t.originalType === type.originalType && t.aliasedType === type.aliasedType,
+          )) {
+            existingModuleImport.types.push(type);
+          }
+        }
+      }
+    }
+  }
+
+  public renderImports() {
+    const sortedImports = this.selectiveImports.sort((a, b) => a.moduleName.localeCompare(b.moduleName));
+    for (const selectiveImport of sortedImports) {
+      const sourceModule = new Module(selectiveImport.moduleName);
+      sourceModule.importSelective(this.module, selectiveImport.types.map((t) => `${t.originalType} as ${t.aliasedType}`), { fromLocation: `${this.modulesRootLocation}/${sourceModule.name}` });
+    }
+  }
 }

tools/@aws-cdk/spec2cdk/lib/cdk/cdk.ts

@@ -22,6 +22,11 @@ export interface ModuleImportLocations {
    * @default 'aws-cdk-lib/aws-cloudwatch'
    */
   readonly cloudwatch?: string;
+  /**
+   * The root location of all the modules
+   * @default '../..'
+   */
+  readonly modulesRoot?: string;
 }
 
 export class CdkCore extends ExternalModule {

tools/@aws-cdk/spec2cdk/lib/cdk/reference-props.ts

@@ -0,0 +1,86 @@
+import { Resource } from '@aws-cdk/service-spec-types';
+import { $E, expr, Expression, PropertySpec, Type } from '@cdklabs/typewriter';
+import { attributePropertyName, referencePropertyName } from '../naming';
+import { CDK_CORE } from './cdk';
+
+export interface ReferenceProp {
+  readonly declaration: PropertySpec;
+  readonly cfnValue: Expression;
+}
+
+// Convenience typewriter builder
+const $this = $E(expr.this_());
+
+export function getReferenceProps(resource: Resource): ReferenceProp[] {
+  const referenceProps = [];
+  // Primary identifier. We assume all parts are strings.
+  const primaryIdentifier = resource.primaryIdentifier ?? [];
+  if (primaryIdentifier.length === 1) {
+    referenceProps.push({
+      declaration: {
+        name: referencePropertyName(primaryIdentifier[0], resource.name),
+        type: Type.STRING,
+        immutable: true,
+        docs: {
+          summary: `The ${primaryIdentifier[0]} of the ${resource.name} resource.`,
+        },
+      },
+      cfnValue: $this.ref,
+    });
+  } else if (primaryIdentifier.length > 1) {
+    for (const [i, cfnName] of enumerate(primaryIdentifier)) {
+      referenceProps.push({
+        declaration: {
+          name: referencePropertyName(cfnName, resource.name),
+          type: Type.STRING,
+          immutable: true,
+          docs: {
+            summary: `The ${cfnName} of the ${resource.name} resource.`,
+          },
+        },
+        cfnValue: splitSelect('|', i, $this.ref),
+      });
+    }
+  }
+
+  const arnProp = findArnProperty(resource);
+  if (arnProp) {
+    referenceProps.push({
+      declaration: {
+        name: referencePropertyName(arnProp, resource.name),
+        type: Type.STRING,
+        immutable: true,
+        docs: {
+          summary: `The ARN of the ${resource.name} resource.`,
+        },
+      },
+      cfnValue: $this[attributePropertyName(arnProp)],
+    });
+  }
+  return referenceProps;
+}
+
+/**
+ * Find an ARN property for a given resource
+ *
+ * Returns `undefined` if no ARN property is found, or if the ARN property is already
+ * included in the primary identifier.
+ */
+export function findArnProperty(resource: Resource) {
+  const possibleArnNames = ['Arn', `${resource.name}Arn`];
+  for (const name of possibleArnNames) {
+    const prop = resource.attributes[name];
+    if (prop && !resource.primaryIdentifier?.includes(name)) {
+      return name;
+    }
+  }
+  return undefined;
+}
+
+function splitSelect(sep: string, n: number, base: Expression) {
+  return CDK_CORE.Fn.select(expr.lit(n), CDK_CORE.Fn.split(expr.lit(sep), base));
+}
+
+function enumerate<A>(xs: A[]): Array<[number, A]> {
+  return xs.map((x, i) => [i, x]);
+}

tools/@aws-cdk/spec2cdk/lib/cdk/relationship-decider.ts

@@ -0,0 +1,172 @@
+import { Property, RelationshipRef, Resource, RichProperty, SpecDatabase } from '@aws-cdk/service-spec-types';
+import { namespaceFromResource, referenceInterfaceName, referenceInterfaceAttributeName, referencePropertyName, typeAliasPrefixFromResource } from '../naming';
+import { getReferenceProps } from './reference-props';
+import { createModuleDefinitionFromCfnNamespace } from '../cfn2ts/pkglint';
+import { log } from '../util';
+
+// For now we want relationships to be applied only for these services
+export const RELATIONSHIP_SERVICES: string[] = [];
+
+/**
+ * Represents a cross-service property relationship that enables references
+ * between resources from different AWS services.
+ */
+export interface Relationship {
+  /** The TypeScript interface type that provides the reference (e.g. "IRoleRef") */
+  readonly referenceType: string;
+  /** The property name on the reference interface that holds the reference object (e.g. "roleRef") */
+  readonly referenceName: string;
+  /** The property to extract from the reference object (e.g. "roleArn") */
+  readonly propName: string;
+}
+
+/**
+ * Represents a selective import statement for cross-module type references.
+ * Used to import specific types from other CDK modules when relationships
+ * are between different modules.
+ */
+export interface SelectiveImport {
+  /** The module name to import from */
+  readonly moduleName: string;
+  /** Array of types that need to be imported */
+  readonly types: {
+    /** The original type name in the source module */
+    originalType: string;
+    /** The aliased name to avoid naming conflicts */
+    aliasedType: string;
+  }[];
+}
+
+/**
+ * Extracts resource relationship information from the database for cross-service property references.
+ */
+export class RelationshipDecider {
+  private readonly namespace: string;
+  public readonly imports = new Array<SelectiveImport>();
+
+  constructor(readonly resource: Resource, private readonly db: SpecDatabase) {
+    this.namespace = namespaceFromResource(resource);
+  }
+
+  private registerRequiredImport({ namespace, originalType, aliasedType }: {
+    namespace: string;
+    originalType: string;
+    aliasedType: string;
+  }) {
+    const moduleName = createModuleDefinitionFromCfnNamespace(namespace).moduleName;
+    const moduleImport = this.imports.find(i => i.moduleName === moduleName);
+    if (!moduleImport) {
+      this.imports.push({
+        moduleName,
+        types: [{ originalType, aliasedType }],
+      });
+    } else {
+      if (!moduleImport.types.find(t =>
+        t.originalType === originalType && t.aliasedType === aliasedType,
+      )) {
+        moduleImport.types.push({ originalType, aliasedType });
+      }
+    }
+  }
+
+  /**
+   * Retrieves the target resource for a relationship.
+   * Returns undefined if the target property cannot be found in the reference
+   * properties as a relationship can only target a primary identifier or arn
+   */
+  private findTargetResource(sourcePropName: string, relationship: RelationshipRef) {
+    if (!RELATIONSHIP_SERVICES.some(s => this.resource.cloudFormationType.toLowerCase().startsWith(`aws::${s}::`))) {
+      return undefined;
+    }
+    const targetResource = this.db.lookup('resource', 'cloudFormationType', 'equals', relationship.cloudFormationType).only();
+    const refProps = getReferenceProps(targetResource);
+    const expectedPropName = referencePropertyName(relationship.propertyName, targetResource.name);
+    const prop = refProps.find(p => p.declaration.name === expectedPropName);
+    if (!prop) {
+      log.debug(
+        'Could not find target prop for relationship:',
+        `${this.resource.cloudFormationType} ${sourcePropName}`,
+        `=> ${targetResource.cloudFormationType} ${relationship.propertyName}`,
+      );
+      return undefined;
+    }
+    return targetResource;
+  }
+
+  public parseRelationship(sourcePropName: string, relationships?: RelationshipRef[]) {
+    const parsedRelationships: Relationship[] = [];
+    if (!relationships) {
+      return parsedRelationships;
+    }
+    for (const relationship of relationships) {
+      const targetResource = this.findTargetResource(sourcePropName, relationship);
+      if (!targetResource) {
+        continue;
+      }
+      // Ignore the suffix part because it's an edge case that happens only for one module
+      const interfaceName = referenceInterfaceName(targetResource.name);
+      const refPropStructName = referenceInterfaceAttributeName(targetResource.name);
+
+      const targetNamespace = namespaceFromResource(targetResource);
+      let aliasedTypeName = undefined;
+      if (this.namespace !== targetNamespace) {
+        // If this is not in our namespace we need to alias the import type
+        aliasedTypeName = `${typeAliasPrefixFromResource(targetResource)}${interfaceName}`;
+        this.registerRequiredImport({ namespace: targetNamespace, originalType: interfaceName, aliasedType: aliasedTypeName });
+      }
+      parsedRelationships.push({
+        referenceType: aliasedTypeName ?? interfaceName,
+        referenceName: refPropStructName,
+        propName: referencePropertyName(relationship.propertyName, targetResource.name),
+      });
+    }
+    return parsedRelationships;
+  }
+
+  /**
+   * Extracts the referenced type from a property's type, for direct refs and array element refs.
+   */
+  private getReferencedType(prop: Property) {
+    // Use the oldest type for backwards compatibility
+    const type = new RichProperty(prop).types()[0];
+    if (type.type === 'ref') {
+      return this.db.get('typeDefinition', type.reference.$ref);
+    } else if (type.type === 'array' && type.element.type === 'ref') {
+      return this.db.get('typeDefinition', type.element.reference.$ref);
+    }
+    return undefined;
+  }
+
+  private hasValidRelationships(sourcePropName: string, relationships?: RelationshipRef[]): boolean {
+    if (!relationships) {
+      return false;
+    }
+    return relationships.some(rel => this.findTargetResource(sourcePropName, rel) !== undefined);
+  }
+
+  /**
+   * Checks if a given property needs a flattening function or not
+   */
+  public needsFlatteningFunction(propName: string, prop: Property, visited = new Set<string>()): boolean {
+    if (this.hasValidRelationships(propName, prop.relationshipRefs)) {
+      return true;
+    }
+
+    const referencedTypeDef = this.getReferencedType(prop);
+    if (!referencedTypeDef) {
+      return false;
+    }
+
+    if (visited.has(referencedTypeDef.$id)) {
+      return false;
+    }
+    visited.add(referencedTypeDef.$id);
+
+    for (const [nestedPropName, nestedProp] of Object.entries(referencedTypeDef.properties)) {
+      if (this.needsFlatteningFunction(nestedPropName, nestedProp, visited)) {
+        return true;
+      }
+    }
+    return false;
+  }
+}