feat: add support for remote templates

docs/working-with-cdk-for-terraform/remote-templates.md

@@ -0,0 +1,32 @@
+# Remote Templates
+
+Templates allow scaffolding a new CDK for Terraform project. When you setup a new project via `cdktf init` you can supply one of the [built-in templates](../packages/cdktf-cli/templates) (e.g. `typescript` or `python`) or use your own. This document describes how to create your own template to use with `cdktf init`.
+
+## Structure of a template
+A template is a directory, containing at least a `cdktf.json` file, which is required for the `cdktf` CLI.  
+For scaffolding a new project the library [`sscaff`](https://github.com/awslabs/node-sscaff) is used. `sscaff` basically copies all files into the new directory while allowing for substitutions and hooks.
+
+### Using Substitutions
+todo: Syntax: `{{}}`
+
+### Using `pre` and `post` Hooks
+todo: what do we do with pre and post hooks?
+
+### Remote Templates
+Currently the `cdktf` supports downloading and extracting a zip archive containing the files for the template. When extracting the archive, it searches for the `cdktf.json` file. If that file cannot be found in the root directory, it walks all directories until it finds the file. This allows creating an archive that contains e.g. a `README.md` in the root directory explaining things which itself won't turn up in the created project directory. However, most templates won't make use of it.  
+
+If you're using a Github repository for your template, you can create URLs to your repo as follows
+#### main branch
+`https://github.com/<user or organization>/<repo>/archive/refs/heads/main.zip`  
+#### tag `v.0.0.1`
+`https://github.com/<user or organization>/<repo>/archive/refs/tags/v0.0.1.zip`
+
+**Please Note:** Currently only public accessible zip archives are supported. If you need support for private packages, please [file an issue](https://github.com/hashicorp/terraform-cdk/issues/new?labels=enhancement%2C+new&template=feature-request.md).
+
+### Debugging
+todo: -> set log output
+
+
+## Remote Template Example
+
+todo: create files and put them in Github repo, run init and supply template via --template.

packages/cdktf-cli/bin/cmds/init.ts

@@ -1,14 +1,18 @@
 import yargs from 'yargs'
 import * as readlineSync from 'readline-sync';
+import extract from 'extract-zip';
 import { TerraformLogin } from './helper/terraform-login'
 import * as fs from 'fs-extra';
+import * as os from 'os';
 import * as path from 'path';
 import { sscaff } from 'sscaff';
 import * as terraformCloudClient from './helper/terraform-cloud-client';
 import * as chalk from 'chalk';
 import { terraformCheck } from './terraform-check';
 import { displayVersionMessage } from './version-check'
 import { FUTURE_FLAGS } from 'cdktf/lib/features';
+import { downloadFile, HttpError } from '../../lib/util';
+import { logger } from '../../lib/logging';
 
 const chalkColour = new chalk.Instance();
 
@@ -28,14 +32,13 @@ class Command implements yargs.CommandModule {
   public readonly describe = 'Create a new cdktf project from a template.';
   public readonly builder = (args: yargs.Argv) => args
     .showHelpOnFail(true)
-    .option('template', { type: 'string', desc: 'The template name to be used to create a new project.' })
-    .option('project-name', { type: 'string', desc: 'The name of the project.'})
-    .option('project-description', { type: 'string', desc: 'The description of the project.'})
+    .option('template', { type: 'string', desc: `The template to be used to create a new project. Either URL to zip file or one of the built-in templates: [${templates.map(t => `"${t}"`).join(', ')}]` })
+    .option('project-name', { type: 'string', desc: 'The name of the project.' })
+    .option('project-description', { type: 'string', desc: 'The description of the project.' })
     .option('dist', { type: 'string', desc: 'Install dependencies from a "dist" directory (for development)' })
-    .option('local', { type: 'boolean', desc: 'Use local state storage for generated Terraform.', default: false})
+    .option('local', { type: 'boolean', desc: 'Use local state storage for generated Terraform.', default: false })
     .option('cdktf-version', { type: 'string', desc: 'The cdktf version to use while creating a new project.', default: pkg.version })
     .strict()
-    .choices('template', templates);
 
   public async handler(argv: any) {
     await terraformCheck()
@@ -65,7 +68,7 @@ This means that your Terraform state file will be stored locally on disk in a fi
     }
 
     // Gather information about the template and the project
-    const templateInfo = await getTemplatePath(template);
+    const templateInfo = await getTemplate(template);
 
     const projectInfo: any = await gatherInfo(token, templateInfo.Name, argv.projectName, argv.projectDescription);
 
@@ -88,6 +91,10 @@ This means that your Terraform state file will be stored locally on disk in a fi
     await sscaff(templateInfo.Path, '.', {
       ...deps, ...projectInfo, futureFlags
     });
+
+    if (templateInfo.cleanupTemporaryFiles) {
+      await templateInfo.cleanupTemporaryFiles()
+    }
   }
 }
 
@@ -185,31 +192,117 @@ If you want to exit, press {magenta ^C}.
 
     console.log(chalkColour`\nWe are going to create a new {blueBright Terraform Cloud Workspace} for your project.\n`)
 
-    const workspaceName = readlineSync.question(chalkColour`{blueBright Terraform Cloud Workspace Name:} (default: '${templateName}') `, { defaultInput: templateName } )
+    const workspaceName = readlineSync.question(chalkColour`{blueBright Terraform Cloud Workspace Name:} (default: '${templateName}') `, { defaultInput: templateName })
     project.OrganizationName = organizationOptions[organizationSelect]
     project.WorkspaceName = workspaceName
   }
 
   return project;
 }
 
-async function getTemplatePath(templateName: string): Promise<Template> {
+/**
+ * 
+ * @param templateName either the name of built-in templates or an url pointing to a zip archive
+ */
+async function getTemplate(templateName: string): Promise<Template> {
   if (templateName == '') {
+    const templateOptionRemote = '<remote zip file>';
+    const options = [...templates, templateOptionRemote];
     // Prompt for template
-    const selection = readlineSync.keyInSelect(templates, chalkColour`{whiteBright What template you want to use?}`)
+    const selection = readlineSync.keyInSelect(options, chalkColour`{whiteBright What template you want to use?}`)
     if (selection == -1) {
       process.exit(0);
     }
-    templateName = templates[selection];
-    console.log(chalkColour`\n{whiteBright Initializing a project using the {greenBright ${templateName}} template.}`);
+    if (selection === options.indexOf(templateOptionRemote)) {
+      templateName = readlineSync.question('Please enter an URL pointing to the template zip file you want to use: ');
+      if (templateName == '') {
+        console.log('No URL was given (received empty string). Aborted.');
+        process.exit(1);
+      }
+    } else {
+      templateName = options[selection];
+      console.log(chalkColour`\n{whiteBright Initializing a project using the {greenBright ${templateName}} template.}`);
+    }
+  }
+
+  // treat as remote url
+  if (!templates.includes(templateName)) {
+    return fetchRemoteTemplate(templateName);
+  } else {
+    return {
+      'Name': templateName,
+      'Path': path.join(templatesDir, templateName)
+    }
   }
+}
 
-  const templatePath = path.join(templatesDir, templateName);
+async function fetchRemoteTemplate(templateUrl: string): Promise<Template> {
+  console.log(chalkColour`Fetching remote template from: {whiteBright ${templateUrl}}`);
+  try {
+    const url = new URL(templateUrl);
+    const remoteFileName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1) || 'template.zip';
+    logger.trace(`Detected remote file name to be "${remoteFileName}" out of template URL "${templateUrl}"`)
+
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdktf.'));
+    const tmpZipFile = path.join(tmpDir, remoteFileName);
+    const zipExtractDir = path.join(tmpDir, 'extracted');
+
+    logger.trace(`Downloading "${remoteFileName}" to temporary directory "${tmpDir}"`);
+    console.log(chalkColour`Downloading "{whiteBright ${remoteFileName}}" to temporary directory`);
+    await downloadFile(url.href, tmpZipFile);
+
+    console.log('Extracting zip file');
+    await extract(tmpZipFile, { dir: zipExtractDir });
+
+    // walk directory to find cdktf.json as the extracted directory contains a root directory with unknown name
+    // this also allows nesting the template itself into a sub directory and having a root directory with an unrelated README
+    console.log(chalkColour`Looking for directory containing {whiteBright cdktf.json}`);
+    const templatePath = await findCdkTfJsonDirectory(zipExtractDir);
+
+    if (!templatePath) {
+      throw new Error(chalkColour`Could not find a {whiteBright cdktf.json} in the extracted directory`);
+    }
 
-  return {
-    'Name': templateName,
-    'Path': templatePath
+    return {
+      'Name': remoteFileName.endsWith('.zip') ? remoteFileName.substring(0, remoteFileName.length - 4) : remoteFileName,
+      'Path': templatePath,
+      cleanupTemporaryFiles: async () => {
+        console.log('Clearing up temporary directory of remote template')
+        await fs.remove(tmpDir);
+      },
+    }
+
+  } catch (e) {
+    if (e.code === 'ERR_INVALID_URL') {
+      console.error(chalkColour`Could not download template: {redBright the supplied url is invalid}`);
+      console.error(chalkColour`Please supply a valid url (including the protocol) or use one of the built-in templates.`);
+      process.exit(1);
+    }
+    if (e instanceof HttpError) {
+      console.error(chalkColour`Could not download template: {redBright ${e.message}}`);
+      process.exit(1);
+    }
+
+    console.error(e)
+    process.exit(1);
+  }
+}
+
+async function findCdkTfJsonDirectory(rootDir: string): Promise<string | null> {
+  const files = await fs.readdir(rootDir);
+
+  if (files.includes('cdktf.json')) {
+    return rootDir;
+  }
+  for (const file of files) {
+    const fullPath = path.join(rootDir, file);
+    if ((await fs.stat(fullPath)).isDirectory()) {
+      const dir = findCdkTfJsonDirectory(fullPath);
+      if (dir) return dir;
+      // else continue with next sub directory
+    }
   }
+  return null;
 }
 
 interface Deps {
@@ -232,6 +325,7 @@ interface Project {
 interface Template {
   Name: string;
   Path: string;
+  cleanupTemporaryFiles?: () => Promise<void>;
 }
 
 module.exports = new Command();

packages/cdktf-cli/lib/util.ts

@@ -1,5 +1,6 @@
 import { spawn, SpawnOptions } from 'child_process';
 import * as fs from 'fs-extra';
+import { https, http } from 'follow-redirects';
 import * as os from 'os';
 import * as path from 'path';
 import { processLogger } from './logging';
@@ -91,4 +92,38 @@ export async function readCDKTFVersion(outputDir: string): Promise<string> {
 export function downcaseFirst(str: string): string {
   if (str === '') { return str; }
   return `${str[0].toLocaleLowerCase()}${str.slice(1)}`;
+}
+
+export class HttpError extends Error {
+  constructor(message?: string, public statusCode?: number) {
+    super(message); // 'Error' breaks prototype chain here
+    Object.setPrototypeOf(this, new.target.prototype); // restore prototype chain
+    // see: https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget
+  }
+}
+
+export async function downloadFile(url: string, targetFilename: string): Promise<void> {
+  const client = url.startsWith('http://') ? http : https;
+  const file = fs.createWriteStream(targetFilename);
+  return new Promise((ok, ko) => {
+    const request = client.get(url, response => {
+      if (response.statusCode !== 200) {
+        ko(new HttpError(`Failed to get '${url}' (${response.statusCode})`, response.statusCode));
+        return;
+      }
+      response.pipe(file);
+    });
+
+    file.on('finish', () => ok());
+
+    request.on('error', err => {
+      fs.unlink(targetFilename, () => ko(err));
+    });
+
+    file.on('error', err => {
+      fs.unlink(targetFilename, () => ko(err));
+    });
+
+    request.end();
+  });
 }

packages/cdktf-cli/package.json

@@ -36,6 +36,8 @@
     "chalk": "^4.1.0",
     "codemaker": "^0.22.0",
     "constructs": "^3.0.0",
+    "extract-zip": "^2.0.1",
+    "follow-redirects": "^1.13.3",
     "fs-extra": "^8.1.0",
     "indent-string": "^4.0.0",
     "ink": "^3.0.8",
@@ -76,6 +78,7 @@
   },
   "devDependencies": {
     "@types/archiver": "^5.1.0",
+    "@types/follow-redirects": "^1.13.0",
     "@types/fs-extra": "^8.1.0",
     "@types/ink": "^2.0.3",
     "@types/ink-spinner": "^3.0.0",

test/csharp/synth-app/test.ts

@@ -8,13 +8,13 @@ import { TestDriver } from "../../test-helper";
 describe("csharp full integration test synth", () => {
   let driver: TestDriver;
 
-  beforeAll(() => {
+  beforeAll(async () => {
     driver = new TestDriver(__dirname)
-    driver.setupCsharpProject()
+    await driver.setupCsharpProject()
   });
 
   test("synth generates JSON", async () => {
-    driver.synth()
+    await driver.synth()
     expect(driver.synthesizedStack()).toMatchSnapshot()
   })
 })

test/java/synth-app/test.ts

@@ -7,14 +7,15 @@ import { TestDriver } from "../../test-helper";
 
 describe("java full integration", () => {
   let driver: TestDriver;
+  jest.setTimeout(60_000);
 
-  beforeAll(() => {
+  beforeAll(async () => {
     driver = new TestDriver(__dirname)
-    driver.setupJavaProject()
+    await driver.setupJavaProject()
   });
 
   test("synth generates JSON", async () => {
-    driver.synth()
+    await driver.synth()
     expect(driver.synthesizedStack()).toMatchSnapshot()
   })
 })

test/package.json

@@ -7,6 +7,7 @@
   "devDependencies": {
     "@skorfmann/terraform-cloud": "^1.9.1",
     "@types/jest": "^26.0.20",
+    "archiver": "^5.3.0",
     "jest": "^26.6.3",
     "jest-runner-groups": "^2.0.1",
     "ts-jest": "^26.5.1",

test/python/synth-app/test.ts

@@ -8,13 +8,13 @@ import { TestDriver } from "../../test-helper";
 describe("python full integration test synth", () => {
   let driver: TestDriver;
 
-  beforeAll(() => {
+  beforeAll(async () => {
     driver = new TestDriver(__dirname)
-    driver.setupPythonProject()
+    await driver.setupPythonProject()
   });
 
   test("synth generates JSON", async () => {
-    driver.synth()
+    await driver.synth()
     expect(driver.synthesizedStack()).toMatchSnapshot()
   })
 })

test/python/third-party-provider/test.ts

@@ -8,13 +8,13 @@ import { TestDriver } from "../../test-helper";
 describe("python full integration 3rd party", () => {
   let driver: TestDriver;
 
-  beforeAll(() => {
+  beforeAll(async () => {
     driver = new TestDriver(__dirname)
-    driver.setupPythonProject()
+    await driver.setupPythonProject()
   });
 
   test("synth generates JSON", async () => {
-    driver.synth()
+    await driver.synth()
     expect(driver.synthesizedStack()).toMatchSnapshot()
   })
 })