diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5317872dc5..5418a8c973 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,8 @@ jobs: terraform: ["0.12.29", "0.13.0"] container: image: hashicorp/jsii-terraform + env: + CHECKPOINT_DISABLE: "1" steps: - uses: actions/checkout@v2 @@ -42,6 +44,8 @@ jobs: container: image: hashicorp/jsii-terraform needs: build + env: + CHECKPOINT_DISABLE: "1" steps: - uses: actions/checkout@v2 @@ -62,6 +66,8 @@ jobs: matrix: terraform: ["0.12.29", "0.13.0"] needs: build + env: + CHECKPOINT_DISABLE: "1" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2919053a3e..0d1be01d5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,8 @@ jobs: runs-on: ubuntu-latest container: image: hashicorp/jsii-terraform + env: + CHECKPOINT_DISABLE: "1" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release_next.yml b/.github/workflows/release_next.yml index cc70bb3e47..c5f2e5b38b 100644 --- a/.github/workflows/release_next.yml +++ b/.github/workflows/release_next.yml @@ -10,6 +10,8 @@ jobs: runs-on: ubuntu-latest container: image: hashicorp/jsii-terraform + env: + CHECKPOINT_DISABLE: "1" steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 181542f2e3..b25afdb3f2 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Choose a language: * Using the CDK for Terraform [tokens](./docs/working-with-cdk-for-terraform/tokens.md). * Using Terraform [data sources](./docs/working-with-cdk-for-terraform/data-sources.md). * Synthesizing Terraform configuration using CDK for Terraform [synthesize](./docs/working-with-cdk-for-terraform/synthesizing-config.md) command. +* Project [telemetry](./docs/working-with-cdk-for-terraform/telemetry.md). ## Contributing and Feedback diff --git a/docs/working-with-cdk-for-terraform/telemetry.md b/docs/working-with-cdk-for-terraform/telemetry.md new file mode 100644 index 0000000000..e43fb63341 --- /dev/null +++ b/docs/working-with-cdk-for-terraform/telemetry.md @@ -0,0 +1,11 @@ +# Telemetry + +CDK for Terraform CLI ([cdktf-cli](../../packages/cdktf-cli)) interacts with a HashiCorp service called [Checkpoint](https://checkpoint.hashicorp.com) +to report project metrics such as cdktf version, project language, provider name, platform name, and other details that help guide the project maintainers with +feature and roadmap decisions. The code that interacts with Checkpoint is part of CDK for Terraform CLI and can be read [here](../../packages/cdktf-cli/bin/lib/checkpoint.ts). + +All HashiCorp projects including Terraform that is used by CDK for Terraform use Checkpoint. +Read more about project metrics [here](https://github.com/hashicorp/terraform-cdk/issues/325). + +The information that is sent to Checkpoint is anonymous and cannot be used to identify the user or host. The use of Checkpoint is completely optional +and it can be disabled at any time by setting the `CHECKPOINT_DISABLE` environment variable to a non-empty value. diff --git a/packages/cdktf-cli/bin/cmds/helper/constructs-maker.ts b/packages/cdktf-cli/bin/cmds/helper/constructs-maker.ts index f28ed4abc9..86db423904 100644 --- a/packages/cdktf-cli/bin/cmds/helper/constructs-maker.ts +++ b/packages/cdktf-cli/bin/cmds/helper/constructs-maker.ts @@ -1,6 +1,7 @@ import { GetProvider } from '../../../lib/get/providers'; import { GetModule } from '../../../lib/get/modules'; import { Language } from '../../../lib/get/base'; +import { Report } from './telemetry'; export interface ConstructsOptions { codeMakerOutput: string; @@ -13,6 +14,7 @@ export class ConstructsMaker { if (modules.length > 0) { await new GetModule().get(Object.assign({}, { codeMakerOutput: constructsOptions.codeMakerOutput, targetLanguage: constructsOptions.language, isModule: true }, { targetNames: modules })); + await moduleTelemetry(constructsOptions.language, modules); } } @@ -20,6 +22,29 @@ export class ConstructsMaker { if (providers.length > 0) { await new GetProvider().get(Object.assign({}, { codeMakerOutput: constructsOptions.codeMakerOutput, targetLanguage: constructsOptions.language }, { targetNames: providers })); + await providerTelemetry(constructsOptions.language, providers); } } } + +async function providerTelemetry(language: string, providers: string[]): Promise { + for (const p of providers) { + const [fqname, version] = p.split('@'); + const name = fqname.split('/').pop() + if (!name) { throw new Error(`Provider name should be properly set in ${p}`) } + + const payload = { name: name, fullName: fqname, version: version, type: 'provider' }; + + await Report('get', language, new Date(), payload); + } +} + +async function moduleTelemetry(language: string, modules: string[]): Promise { + for (const module of modules) { + const [source, version] = module.split('@'); + + const payload = { source: source, version: version, type: 'module' }; + + await Report('get', language, new Date(), payload) + } +} diff --git a/packages/cdktf-cli/bin/cmds/helper/synth-stack.ts b/packages/cdktf-cli/bin/cmds/helper/synth-stack.ts index 5e922dcc3b..b0c0f12faa 100644 --- a/packages/cdktf-cli/bin/cmds/helper/synth-stack.ts +++ b/packages/cdktf-cli/bin/cmds/helper/synth-stack.ts @@ -2,6 +2,8 @@ import { shell } from '../../../lib/util'; import * as fs from 'fs-extra'; import * as path from 'path' import { TerraformStackMetadata } from 'cdktf' +import { Report } from './telemetry'; +import { performance } from 'perf_hooks'; interface SynthesizedStackMetadata { "//"?: {[key: string]: TerraformStackMetadata }; @@ -15,6 +17,9 @@ interface SynthesizedStack { export class SynthStack { public static async synth(command: string, outdir: string): Promise { + // start performance timer + const startTime = performance.now(); + await shell(command, [], { shell: true, env: { @@ -28,6 +33,10 @@ export class SynthStack { process.exit(1); } + // end performance timer + const endTime = performance.now(); + await this.synthTelemetry(command, (endTime - startTime)); + const stacks: SynthesizedStack[] = []; for (const file of await fs.readdir(outdir)) { @@ -55,4 +64,10 @@ export class SynthStack { return stacks } -} + + public static async synthTelemetry(command: string, totalTime: number): Promise { + const payload = { command: command, totalTime: totalTime }; + + await Report('synth', '', new Date(), payload); + } +} \ No newline at end of file diff --git a/packages/cdktf-cli/bin/cmds/helper/telemetry.ts b/packages/cdktf-cli/bin/cmds/helper/telemetry.ts new file mode 100644 index 0000000000..d4049840d1 --- /dev/null +++ b/packages/cdktf-cli/bin/cmds/helper/telemetry.ts @@ -0,0 +1,16 @@ +import { ReportParams, ReportRequest } from '../../../lib/checkpoint' +import { versionNumber } from '../version-check'; +import { readConfigSync } from '../../../lib/config'; + +const product = "cdktf" +const config = readConfigSync() + +export async function Report(command: string, language: string, dateTime: Date, payload: {}): Promise { + if (language == '') { + if (config.language) { + language = config.language + } + } + const reportParams: ReportParams = { command: command, product: product, version: versionNumber(), dateTime: dateTime, payload: payload, language: language }; + await ReportRequest(reportParams); +} \ No newline at end of file diff --git a/packages/cdktf-cli/bin/cmds/ui/models/terraform.ts b/packages/cdktf-cli/bin/cmds/ui/models/terraform.ts index d612a1bf9e..792a1b4762 100644 --- a/packages/cdktf-cli/bin/cmds/ui/models/terraform.ts +++ b/packages/cdktf-cli/bin/cmds/ui/models/terraform.ts @@ -83,12 +83,7 @@ export class Terraform { } public async init(): Promise { - // Read the cdktf version from the 'cdk.tf.json' file - // and set the user agent. - const version = await readCDKTFVersion(this.workdir) - if (version != "") { - process.env.TF_APPEND_USER_AGENT = "cdktf " + version + " (+https://github.com/hashicorp/terraform-cdk)"; - } + await this.setUserAgent() await exec(terraformBinaryName, ['init'], { cwd: this.workdir, env: process.env }) } @@ -98,6 +93,7 @@ export class Terraform { if (destroy) { options.push('-destroy') } + await this.setUserAgent() await exec(terraformBinaryName, options, { cwd: this.workdir, env: process.env }); const jsonPlan = await exec(terraformBinaryName, ['show', '-json', planFile], { cwd: this.workdir, env: process.env }); return new TerraformPlan(planFile, JSON.parse(jsonPlan)); @@ -105,10 +101,12 @@ export class Terraform { public async deploy(planFile: string, stdout: (chunk: Buffer) => any): Promise { const relativePlanFile = path.relative(this.workdir, planFile); + await this.setUserAgent() await exec(terraformBinaryName, ['apply', '-auto-approve', ...this.stateFileOption, relativePlanFile], { cwd: this.workdir, env: process.env }, stdout); } public async destroy(stdout: (chunk: Buffer) => any): Promise { + await this.setUserAgent() await exec(terraformBinaryName, ['destroy', '-auto-approve', ...this.stateFileOption], { cwd: this.workdir, env: process.env }, stdout); } @@ -127,4 +125,13 @@ export class Terraform { private get stateFileOption() { return ['-state', path.join(process.cwd(), 'terraform.tfstate')] } + + public async setUserAgent(): Promise { + // Read the cdktf version from the 'cdk.tf.json' file + // and set the user agent. + const version = await readCDKTFVersion(this.workdir) + if (version != "") { + process.env.TF_APPEND_USER_AGENT = "cdktf/" + version + " (+https://github.com/hashicorp/terraform-cdk)"; + } + } } diff --git a/packages/cdktf-cli/lib/checkpoint.ts b/packages/cdktf-cli/lib/checkpoint.ts new file mode 100644 index 0000000000..8012edf811 --- /dev/null +++ b/packages/cdktf-cli/lib/checkpoint.ts @@ -0,0 +1,88 @@ +import https = require('https'); +import { format } from 'url'; +import { v4 as uuidv4 } from 'uuid'; +import * as os from 'os'; +import { processLogger } from './logging'; + +const BASE_URL = `https://checkpoint-api.hashicorp.com/v1/`; + +const VALID_STATUS_CODES = [200, 201]; + +export interface ReportParams { + dateTime?: Date; + arch?: string; + os?: string; + payload: {}; + product: string; + runID?: string; + version?: string; + command?: string; + language?: string; +} + +async function post(url: string, data: string) { + return new Promise((ok, ko) => { + const req = https.request(format(url), { + headers: { + 'Accept': 'application/json', + 'Content-Length': data.length, + 'User-Agent': 'HashiCorp/cdktf-cli' + }, + method: 'POST' + }, res => { + if (res.statusCode) { + const statusCode = res.statusCode; + if (!VALID_STATUS_CODES.includes(statusCode)) { + return ko(new Error(res.statusMessage)); + } + } + const data = new Array(); + res.on('data', chunk => data.push(chunk)); + + res.once('error', err => ko(err)); + res.once('end', () => { + return ok(); + }); + }); + + req.write(data); + + req.end(); + }) +} + +export async function ReportRequest(reportParams: ReportParams): Promise { + // we won't report when checkpoint is disabled. + if (process.env.CHECKPOINT_DISABLE) { + return + } + + if (!reportParams.runID) { + reportParams.runID = uuidv4(); + } + + if (!reportParams.dateTime) { + reportParams.dateTime = new Date(); + } + + if (!reportParams.arch) { + reportParams.arch = os.arch(); + } + + if (!reportParams.os) { + reportParams.os = os.platform(); + } + + const postData = JSON.stringify(reportParams); + + try { + await post(`${BASE_URL}telemetry/${reportParams.product}`, postData) + } catch (e) { + // Log errors writing to checkpoint + processLogger(e.message) + } + +} + + + diff --git a/packages/cdktf-cli/package.json b/packages/cdktf-cli/package.json index 3f80f66672..c96dfdfcff 100644 --- a/packages/cdktf-cli/package.json +++ b/packages/cdktf-cli/package.json @@ -28,6 +28,7 @@ "dependencies": { "@types/node": "^14.0.26", "@types/readline-sync": "^1.4.3", + "@types/uuid": "^8.3.0", "cdktf": "0.0.0", "chalk": "^4.1.0", "codemaker": "^0.22.0", @@ -44,6 +45,7 @@ "readline-sync": "^1.4.10", "semver": "^7.3.2", "sscaff": "^1.2.0", + "uuid": "^8.3.0", "yargs": "^15.1.0" }, "eslintConfig": { @@ -85,4 +87,4 @@ "ts-jest": "^25.4.0", "typescript": "^3.9.7" } -} \ No newline at end of file +} diff --git a/test/test-checkpoint-service/expected/cdk.tf.json b/test/test-checkpoint-service/expected/cdk.tf.json new file mode 100644 index 0000000000..92581685c7 --- /dev/null +++ b/test/test-checkpoint-service/expected/cdk.tf.json @@ -0,0 +1,8 @@ +{ + "//": { + "metadata": { + "version": "stubbed", + "stackName": "hello-terra" + } + } +} \ No newline at end of file diff --git a/test/test-checkpoint-service/main.ts b/test/test-checkpoint-service/main.ts new file mode 100644 index 0000000000..f5f1d83efb --- /dev/null +++ b/test/test-checkpoint-service/main.ts @@ -0,0 +1,15 @@ +import { Construct } from 'constructs'; +import { App, TerraformStack, Testing } from 'cdktf'; + +export class HelloTerra extends TerraformStack { + constructor(scope: Construct, id: string) { + super(scope, id); + + // define resources here + + } +} + +const app = Testing.stubVersion(new App({stackTraces: false})); +new HelloTerra(app, 'hello-terra'); +app.synth(); \ No newline at end of file diff --git a/test/test-checkpoint-service/test.sh b/test/test-checkpoint-service/test.sh new file mode 100755 index 0000000000..fa1d4a0ffc --- /dev/null +++ b/test/test-checkpoint-service/test.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -e +scriptdir=$(cd $(dirname $0) && pwd) + +cd $(mktemp -d) +mkdir test && cd test + +# hidden files should be ignored +touch .foo +mkdir .bar + +# unset CHECKPOINT_DISABLE +export CHECKPOINT_DISABLE="" + +# initialize an empty project +cdktf init --template typescript --project-name="typescript-test" --project-description="typescript test app" --local + +# put some code in it +cp ${scriptdir}/main.ts . + +# build +yarn compile +yarn synth > /dev/null + +# get rid of downloaded Terraform providers, no point in diffing them +rm -rf cdktf.out/.terraform + +# show output +diff cdktf.out ${scriptdir}/expected + +echo "PASS" \ No newline at end of file