From b159a9dff319f3b34c1af79277c8e41a56c9acd7 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 22 Jan 2024 07:13:30 +1100 Subject: [PATCH] upload .codegenie and download project --- .eslintrc.json | 2 + package.json | 2 + src/app-definition-generator.test.ts | 18 +-- src/app-definition-generator.ts | 8 +- src/commands/generate.ts | 160 +++++++++++++++++++++++---- src/sleep.ts | 4 + 6 files changed, 157 insertions(+), 37 deletions(-) create mode 100644 src/sleep.ts diff --git a/.eslintrc.json b/.eslintrc.json index 95b54cb..ff2f56b 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -11,6 +11,8 @@ "perfectionist/sort-object-types": "off", "jsdoc/require-jsdoc": "off", "jsdoc/require-returns": "off", + "jsdoc/require-param-description": "off", + "jsdoc/require-param-type": "off", "@typescript-eslint/no-explicit-any": "off" } } diff --git a/package.json b/package.json index c4f5237..4fbe562 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@oclif/plugin-plugins": "^4", "@oclif/plugin-update": "^4.1.7", "@oclif/plugin-warn-if-update-available": "^3.0.9", + "archiver": "^6.0.1", "axios": "^1.6.5", "cosmiconfig": "^9.0.0", "js-yaml": "^4.1.0", @@ -46,6 +47,7 @@ "devDependencies": { "@oclif/prettier-config": "^0.2.1", "@oclif/test": "^3", + "@types/archiver": "^6.0.2", "@types/chai": "^4", "@types/js-yaml": "^4.0.9", "@types/mocha": "^10", diff --git a/src/app-definition-generator.test.ts b/src/app-definition-generator.test.ts index 67e2e47..c670596 100644 --- a/src/app-definition-generator.test.ts +++ b/src/app-definition-generator.test.ts @@ -10,7 +10,7 @@ describe('main', () => { expect(entitySchemas).equal([ { entityName: 'App', - fileName: 'app.yaml', + fileName: 'app.yml', jsonSchema: { $schema: 'http://json-schema.org/draft-07/schema#', title: 'App', @@ -20,8 +20,8 @@ describe('main', () => { dynamodb: { partitionKey: 'userId', sortKey: 'appId' }, isRootEntity: true, hasMany: { - Entity: { $ref: './entity.yaml' }, - Build: { $ref: './build.yaml' }, + Entity: { $ref: './entity.yml' }, + Build: { $ref: './build.yml' }, }, }, allOf: [{ type: 'object', $ref: '#/definitions/attributes' }], @@ -56,7 +56,7 @@ describe('main', () => { }, { entityName: 'Entity', - fileName: 'entity.yaml', + fileName: 'entity.yml', jsonSchema: { $schema: 'http://json-schema.org/draft-07/schema#', title: 'Entity', @@ -65,8 +65,8 @@ describe('main', () => { nameProperty: 'name', dynamodb: { partitionKey: 'appId', sortKey: 'entityId' }, hasMany: { - Property: { $ref: './property.yaml' }, - Many: { $ref: './many.yaml' }, + Property: { $ref: './property.yml' }, + Many: { $ref: './many.yml' }, }, }, allOf: [{ type: 'object', $ref: '#/definitions/attributes' }], @@ -96,7 +96,7 @@ describe('main', () => { }, { entityName: 'Property', - fileName: 'property.yaml', + fileName: 'property.yml', jsonSchema: { $schema: 'http://json-schema.org/draft-07/schema#', title: 'Property', @@ -134,7 +134,7 @@ describe('main', () => { }, { entityName: 'Many', - fileName: 'many.yaml', + fileName: 'many.yml', jsonSchema: { $schema: 'http://json-schema.org/draft-07/schema#', title: 'Many', @@ -166,7 +166,7 @@ describe('main', () => { }, { entityName: 'Build', - fileName: 'build.yaml', + fileName: 'build.yml', jsonSchema: { $schema: 'http://json-schema.org/draft-07/schema#', title: 'Build', diff --git a/src/app-definition-generator.ts b/src/app-definition-generator.ts index 8c09b63..75f6b3b 100644 --- a/src/app-definition-generator.ts +++ b/src/app-definition-generator.ts @@ -97,13 +97,13 @@ function getDefaultAuthRoute({ app }: { app: App }) { } function writeAppYamlToFileSystem({ app, appName, appDescription }: { app: App; appName: string; appDescription: string }) { - const fileName = 'app.yaml' + const fileName = 'app.yml' const entities: string[] = app.entities.map((entity) => _s.camelize(_s.decapitalize(entity.name))) const defaultAuthRoute = getDefaultAuthRoute({ app }) const definitions: any = {} for (const entity of app.entities) { definitions[_s.camelize(_s.decapitalize(entity.name))] = { - $ref: `./entities/${paramCase(entity.name)}.yaml`, + $ref: `./entities/${paramCase(entity.name)}.yml`, } } @@ -127,7 +127,7 @@ function writeAppYamlToFileSystem({ app, appName, appDescription }: { app: App; export function getJsonSchemasFromEntities({ app }: { app: App }) { return app.entities.map((entity) => { const paramCasedEntityName = paramCase(entity.name) - const fileName = `${paramCasedEntityName}.yaml` + const fileName = `${paramCasedEntityName}.yml` const codeGenieEntityJsonSchema = convertToCodeGenieEntityJsonSchema({ app, entity, @@ -267,7 +267,7 @@ function getHasManySettings({ app, entity }: { app: App; entity: AppEntity }) { for (const appEntity of app.entities) { if (appEntity.belongsTo === entity.name) { hasMany[appEntity.name] = { - $ref: `./${paramCase(appEntity.name)}.yaml`, + $ref: `./${paramCase(appEntity.name)}.yml`, } } } diff --git a/src/commands/generate.ts b/src/commands/generate.ts index e58ae0e..9b18dc3 100644 --- a/src/commands/generate.ts +++ b/src/commands/generate.ts @@ -1,11 +1,13 @@ -import { existsSync, rmSync } from 'node:fs' +import { createReadStream, createWriteStream, existsSync, rmSync, statSync } from 'node:fs' import { Command, Flags, ux } from '@oclif/core' -import axios from 'axios' +import axios, { AxiosResponse } from 'axios' +import archiver from 'archiver' import { cosmiconfig } from 'cosmiconfig' - import codeGenieSampleOpenAiOutputJson from '../sample-api-output.js' import { App, convertOpenAiOutputToCodeGenieInput } from '../app-definition-generator.js' import copyAwsProfile from '../copyAwsProfile.js' +import path from 'node:path' +import sleep from '../sleep.js' const explorer = cosmiconfig('codegenie', { searchPlaces: [ `.codegenie/app`, @@ -27,7 +29,8 @@ const explorer = cosmiconfig('codegenie', { ], }) -axios.defaults.baseURL = 'http://localhost:4911' +// axios.defaults.baseURL = 'http://localhost:4911' +axios.defaults.baseURL = 'https://r0jmyp0py4.execute-api.us-west-2.amazonaws.com' export default class Generate extends Command { public static enableJsonFlag = true @@ -81,18 +84,24 @@ generating app... const { flags } = await this.parse(Generate) const { description, deploy, awsProfileToCopy, noCopyAwsProfile } = flags + if (description && description.length > 500) { + this.error('description must be less than 500 characters.', { + code: 'DESCRIPTION_TOO_LONG', + suggestions: ['Try again with a shorter description.'], + }) + } + // If a description is provided we generate an App Definition .codegenie directory based on it if (description) { await this.handleExistingAppDefinition() await this.generateAppDefinition() } - // TODO: If no appId defined in app.yaml, create App in DB and update app.yaml with appId. - - // TODO: Create Build in DB with current timestamp. Returns presigned URL for uploading. - - await this.uploadAppDefinition() - await this.pollForS3OutputObject() + const { headObjectPresignedUrl, getObjectPresignedUrl } = await this.uploadAppDefinition() + await this.downloadS3OutputObject({ + headObjectPresignedUrl, + getObjectPresignedUrl, + }) if (!noCopyAwsProfile) { const appDefinition = await explorer.search() @@ -144,7 +153,7 @@ generating app... /** * Generates a [.codegenie app definition](https://codegenie.codes/docs) based on the provided description */ - async generateAppDefinition(): Promise { + async generateAppDefinition(): Promise { const { flags } = await this.parse(Generate) const { description, name } = flags // this.log(`Generating .codegenie app definition based on the following description: @@ -152,12 +161,24 @@ generating app... // ${description} // `) ux.action.start('🧞 Generating App Definition') - // const output = await axios.post('/app-definition-generator', { - // name, - // description, - // }) - // const app = output.data.data - const app = codeGenieSampleOpenAiOutputJson as unknown as App + const output = await axios.post('/app-definition-generator', { + name, + description, + }) + const app = output.data.data + // const app = codeGenieSampleOpenAiOutputJson as unknown as App + + if (!app) { + this.error("The Genie couldn't grant your wish.", { + code: 'GENERATE_APP_DEFINITION_FAILED', + suggestions: [ + 'Try again with a different description.', + 'Report the error in the Code Genie Discord Server listed on https://codegenie.codes or contact support@codegenie.codes.', + ], + }) + return + } + convertOpenAiOutputToCodeGenieInput({ app, appName: name, @@ -167,24 +188,115 @@ generating app... return app } + async createZip(directoryPath: string, zipFilePath: string) { + return new Promise((resolve, reject) => { + const output = createWriteStream(zipFilePath) + const archive = archiver('zip') + + output.on('close', () => { + resolve() + }) + + archive.on('error', (err) => { + reject(err) + }) + + archive.pipe(output) + + // Append the entire directory to the archive + archive.directory(directoryPath, false) + + archive.finalize() + }) + } + /** * Uploads App Definition .codegenie directory to S3, which kicks off an app build */ async uploadAppDefinition() { ux.action.start('⬆️📦 Uploading App Definition') - // (1MB size limit) to S3: apps/appId/timestamp/input.zip (use timestamp from Build record) - // S3 triggers Lambda to run generator and uploads generated output zip to S3: apps/appId/timestamp/output.zip - // ux.action.status = 'still going' + const output = await axios.get('/build-upload-presigned-url') + const { putObjectPresignedUrl, headObjectPresignedUrl, getObjectPresignedUrl } = output.data.data + const zipFilePath = path.join('.codegenie', '.codegenie.zip') + + await this.createZip('.codegenie', zipFilePath) + + const zipFileStats = statSync(zipFilePath) + const zipFileSizeBytes = zipFileStats.size + const fileStream = createReadStream(zipFilePath) + + await axios.put(putObjectPresignedUrl, fileStream, { + headers: { + 'Content-Type': 'application/zip', + 'Content-Length': zipFileSizeBytes.toString(), + }, + }) + ux.action.stop('✅') + return { + headObjectPresignedUrl, + getObjectPresignedUrl, + } } - /** - * TODO: - */ - async pollForS3OutputObject(): Promise { + async pollS3ObjectExistence({ headObjectPresignedUrl, startTime = Date.now() }: { headObjectPresignedUrl: string; startTime?: number }) { + const timeout = 60_000 + const interval = 2000 + + // If timeout is reached, stop polling + if (Date.now() - startTime >= timeout) { + this.error('Timed out waiting for app to generate.', { + code: 'DESCRIPTION_TOO_LONG', + suggestions: ['Try again with a shorter description.'], + }) + } + + try { + // Make a HEAD request to check the existence of the S3 object + const response = await axios.head(headObjectPresignedUrl) + + // If the response status is 200 (OK), the object exists + if (response.status === 200) { + return + } + + this.error(`Received unexpected status code ${response.status} while checking if the app had finished generating.`, { + code: 'POLLING_APP_UNEXPECTED_STATUS_CODE', + suggestions: ['Try again.', 'Report error to discord server or GitHub'], + }) + } catch (error: any) { + console.log(error, typeof error) + if ((error.response as AxiosResponse)?.status === 404) { + // Wait for the specified interval before making the next request + sleep(interval) + + // Start the recursive polling + await this.pollS3ObjectExistence({ headObjectPresignedUrl, startTime }) + return + } + + // Handle errors, e.g., log or ignore + console.error(error) + this.error(`Received unexpected error while checking if the app had finished generating.`, { + code: 'POLLING_APP_UNEXPECTED_ERROR', + suggestions: ['Try again.', 'Report error to discord server or GitHub'], + }) + } + } + + async downloadS3OutputObject({ + headObjectPresignedUrl, + getObjectPresignedUrl, + }: { + headObjectPresignedUrl: string + getObjectPresignedUrl: string + }): Promise { ux.action.start('🏗️ Generating project') + await this.pollS3ObjectExistence({ headObjectPresignedUrl }) ux.action.stop('✅') ux.action.start('⬇️📦 Downloading project') + const response = await axios.get(getObjectPresignedUrl) + console.log(response) ux.action.stop('✅') // Polls API /apps/appId/builds/buildId for existence. Returns presigned URL to download output.zip from S3 and extracts. } diff --git a/src/sleep.ts b/src/sleep.ts new file mode 100644 index 0000000..42f615f --- /dev/null +++ b/src/sleep.ts @@ -0,0 +1,4 @@ +export default function sleep(ms: number) { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => setTimeout(resolve, ms)) +}