fix(lib): Add struct sharding for providers without namespaces

packages/@cdktf/provider-generator/lib/get/__tests__/generator/export-sharding.test.ts

@@ -127,3 +127,43 @@ test("shard exports across multiple files to avoid generating files with more th
  );
  expect(outputStructs3600).toMatchSnapshot(`structs3600`);
 });
+
+test("shard exports across multiple files to avoid generating files with more than a 1000 exports in a provider without namespaces", async () => {
+ const code = new CodeMaker();
+ const workdir = fs.mkdtempSync(
+ path.join(os.tmpdir(), "export-sharding-no-namespace.test")
+ );
+
+ const spec = JSON.parse(
+ fs.readFileSync(
+ path.join(__dirname, "fixtures", "datadog_dashboard.test.fixture.json"),
+ "utf-8"
+ )
+ );
+ new TerraformProviderGenerator(code, spec);
+ await code.save(workdir);
+
+ const output = fs.readFileSync(
+ path.join(workdir, "providers/datadog/dashboard.ts"),
+ "utf-8"
+ );
+ expect(output).toMatchSnapshot(`dashboard-resource`);
+
+ const outputStructsIndex = fs.readFileSync(
+ path.join(workdir, "providers/datadog/dashboard-structs/index.ts"),
+ "utf-8"
+ );
+ expect(outputStructsIndex).toMatchSnapshot(`structs-index`);
+
+ const outputStructs0 = fs.readFileSync(
+ path.join(workdir, "providers/datadog/dashboard-structs/structs0.ts"),
+ "utf-8"
+ );
+ expect(outputStructs0).toMatchSnapshot(`structs0`);
+
+ const outputStructs400 = fs.readFileSync(
+ path.join(workdir, "providers/datadog/dashboard-structs/structs400.ts"),
+ "utf-8"
+ );
+ expect(outputStructs400).toMatchSnapshot(`structs400`);
+});

packages/@cdktf/provider-generator/lib/get/__tests__/provider.test.ts

@@ -14,3 +14,5 @@ getProvider(
 );
 
 getProvider(new TerraformProviderConstraint("oci@= 4.13.0"));
+
+getProvider(new TerraformProviderConstraint("DataDog/datadog@= 3.6.0"));

packages/@cdktf/provider-generator/lib/get/__tests__/util.ts

@@ -10,7 +10,7 @@ import {
 export function expectImportMatchSnapshot(
  constraint: TerraformDependencyConstraint
 ) {
- jest.setTimeout(120_000);
+ jest.setTimeout(240_000);
 
  test(constraint.name, async () => {
  await mkdtemp(async (workdir) => {

packages/@cdktf/provider-generator/lib/get/generator/emitter/struct-emitter.ts

@@ -3,7 +3,7 @@ import { ResourceModel, Struct, ConfigStruct } from "../models";
 import { AttributesEmitter } from "./attributes-emitter";
 import { downcaseFirst } from "../../../util";
 import * as path from "path";
-import { STRUCT_NAMESPACE_THRESHOLD } from "../models/resource-model";
+import { STRUCT_SHARDING_THRESHOLD } from "../models/resource-model";
 export class StructEmitter {
  attributesEmitter: AttributesEmitter;
 
@@ -12,7 +12,7 @@ export class StructEmitter {
  }
 
  public emit(resource: ResourceModel) {
- if (resource.structsRequireNamespace) {
+ if (resource.structsRequireSharding) {
  this.emitNamespacedStructs(resource);
  } else {
  this.emitStructs(resource);
@@ -84,19 +84,15 @@ export class StructEmitter {
  for (
  let i = 0;
  i < structsWithoutConfigStruct.length;
- i += STRUCT_NAMESPACE_THRESHOLD
+ i += STRUCT_SHARDING_THRESHOLD
  ) {
  const structsToImport: Record<string, string[]> = {};
  const structs = structsWithoutConfigStruct.slice(
  i,
- i + STRUCT_NAMESPACE_THRESHOLD
+ i + STRUCT_SHARDING_THRESHOLD
  );
  const structFilename = `structs${i}.ts`;
  structPaths.push(structFilename);
- const namespacedFilePath = path.join(
- resource.namespacedFilePath,
- structFilename
- );
 
  // find all structs that need to be imported in this file
  structs.forEach((struct) => {
@@ -131,6 +127,11 @@ export class StructEmitter {
  // to find it in subsequent files for importing
  structs.map((struct) => (structImports[struct.name] = structFilename));
 
+ const namespacedFilePath = path.join(
+ resource.structsFolderPath,
+ structFilename
+ );
+
  this.code.openFile(namespacedFilePath);
  // the structs only makes use of cdktf not constructs
  this.code.line(`import * as cdktf from 'cdktf';`);
@@ -161,7 +162,7 @@ export class StructEmitter {
  }
 
  // emit the index file that exports all the struct files we've just generated
- const indexFilePath = path.join(resource.namespacedFilePath, "index.ts");
+ const indexFilePath = path.join(resource.structsFolderPath, "index.ts");
 
  this.code.openFile(indexFilePath);
  structPaths.forEach((structPath) => {

packages/@cdktf/provider-generator/lib/get/generator/models/resource-model.ts

@@ -9,7 +9,7 @@ import { Schema } from "../provider-schema";
 import { AttributeModel } from "./attribute-model";
 import { Struct, ConfigStruct } from "./struct";
 
-export const STRUCT_NAMESPACE_THRESHOLD = 400;
+export const STRUCT_SHARDING_THRESHOLD = 400;
 interface ResourceModelOptions {
  terraformType: string;
  className: string;
@@ -58,8 +58,8 @@ export class ResourceModel {
  ];
  }
 
- public get structsRequireNamespace(): boolean {
- return this._structs.length > STRUCT_NAMESPACE_THRESHOLD;
+ public get structsRequireSharding(): boolean {
+ return this._structs.length > STRUCT_SHARDING_THRESHOLD;
  }
 
  public get structs(): Struct[] {
@@ -177,12 +177,14 @@ export class ResourceModel {
  .map((a) => a.type.typeName);
  }
 
- public get namespacedFilePath(): string {
- return path.join(
- this.filePath.split("/").slice(0, -1).join("/"),
- this.namespace!.name,
- this.structsFolderName
- );
+ public get structsFolderPath(): string {
+ const basePath = this.filePath.split("/").slice(0, -1).join("/");
+
+ if (this.namespace) {
+ return path.join(basePath, this.namespace!.name, this.structsFolderName);
+ } else {
+ return path.join(basePath, this.structsFolderName);
+ }
  }
 
  private escapeSchema(schema: string): string {

packages/@cdktf/provider-generator/lib/get/generator/provider-generator.ts

@@ -102,6 +102,8 @@ export class TerraformProviderGenerator {
  namespacedResources[namespace] = [];
  }
  namespacedResources[namespace].push(resourceModel);
+ } else if (resourceModel.structsRequireSharding) {
+ files.push(this.emitResourceFileWithComplexStruct(resourceModel));
  } else {
  files.push(this.emitResourceFile(resourceModel));
  }
@@ -187,7 +189,7 @@ export class TerraformProviderGenerator {
  this.code.line(`// generated from terraform resource schema`);
  this.code.line();
 
- if (resource.structsRequireNamespace) {
+ if (resource.structsRequireSharding) {
  this.code.line(
  `import { ${resource.importableTypes.join(", \n")}} from './${
  resource.structsFolderName
@@ -253,6 +255,68 @@ export class TerraformProviderGenerator {
  return `ns:${ns.name}`;
  }
 
+ private emitResourceFileWithComplexStruct(resource: ResourceModel) {
+ const generatedFiles = [];
+
+ // drop the last segment of the filepath
+ const filePath = resource.filePath;
+ this.code.openFile(filePath);
+ this.code.line(`// generated from terraform resource schema`);
+ this.code.line();
+
+ if (resource.structsRequireSharding) {
+ if (resource.importableTypes.length > 0) {
+ this.code.line(
+ `import { ${resource.importableTypes.join(", \n")}} from './${
+ resource.structsFolderName
+ }'`
+ );
+ }
+
+ if (resource.importableStructMapper.length > 0) {
+ this.code.line(
+ `import { ${resource.importableStructMapper.join(", \n")}} from './${
+ resource.structsFolderName
+ }'`
+ );
+ }
+
+ if (resource.importableOutputReferences.length > 0) {
+ this.code.line(
+ `import { ${resource.importableOutputReferences.join(
+ ",\n"
+ )} } from './${resource.structsFolderName}'`
+ );
+ }
+
+ this.code.line(`export * from './${resource.structsFolderName}'`);
+
+ resource.importStatements.forEach((statement) =>
+ this.code.line(statement)
+ );
+
+ this.structEmitter.emitInterface(resource, resource.configStruct);
+ this.resourceEmitter.emit(resource);
+
+ this.code.closeFile(filePath);
+
+ this.structEmitter.emit(resource);
+ generatedFiles.push(resource.fileName);
+ generatedFiles.push(resource.structsFolderName);
+ } else {
+ resource.importStatements.forEach((statement) =>
+ this.code.line(statement)
+ );
+
+ this.structEmitter.emit(resource);
+ this.resourceEmitter.emit(resource);
+ this.code.closeFile(filePath);
+ generatedFiles.push(resource.fileName);
+ }
+
+ return filePath;
+ }
+
  private emitFileHeader(resource: ResourceModel) {
  this.code.line(`// ${resource.linkToDocs}`);
  this.code.line(`// generated from terraform resource schema`);

test/typescript/providers/cdktf.json

@@ -12,7 +12,8 @@
  "nomad",
  "external",
  "terraform-providers/openstack",
- "oci"
+ "oci",
+ "DataDog/datadog@~> 3.0"
  ],
  "context": {
  "excludeStackIdFromLogicalIds": "true"

test/typescript/providers/main.ts

@@ -16,6 +16,7 @@ import * as Nomad from "./.gen/providers/nomad";
 import * as Vault from "./.gen/providers/vault";
 import * as Consul from "./.gen/providers/consul";
 import * as External from "./.gen/providers/external";
+import * as Datadog from "./.gen/providers/datadog";
 
 export class UsingAllProviders extends TerraformStack {
  constructor(scope: Construct, id: string) {
@@ -32,7 +33,18 @@ export class UsingAllProviders extends TerraformStack {
  },
  ]);
 
- [Aws, Azure, Google, Kubernetes, Nomad, Vault, Openstack, Consul, External];
+ [
+ Aws,
+ Azure,
+ Google,
+ Kubernetes,
+ Nomad,
+ Vault,
+ Openstack,
+ Consul,
+ External,
+ Datadog,
+ ];
  }
 }