From c9165a071b730d8849f5d0d5ff8756614a6fd6de Mon Sep 17 00:00:00 2001 From: Ryan Graham Date: Fri, 22 May 2020 09:21:58 -0400 Subject: [PATCH] original without streams --- app/apollo/resolvers/channel.js | 91 +++++++++---------- app/conf.js | 5 +- app/s3/s3Client.js | 151 +++++++++++++++++++++++++++++++- 3 files changed, 196 insertions(+), 51 deletions(-) diff --git a/app/apollo/resolvers/channel.js b/app/apollo/resolvers/channel.js index a1f136240..ea3206ee8 100644 --- a/app/apollo/resolvers/channel.js +++ b/app/apollo/resolvers/channel.js @@ -17,6 +17,8 @@ const _ = require('lodash'); const { v4: UUID } = require('uuid'); const crypto = require('crypto'); +const conf = require('../../conf.js').conf; +const S3ClientClass = require('../../s3/s3Client'); const { ACTIONS, TYPES } = require('../models/const'); const { whoIs, validAuth } = require ('./common'); @@ -73,17 +75,29 @@ const channelResolvers = { throw `versionObj "${version_uuid}" is not found for ${channel.name}:${channel.uuid}.`; } - if (versionObj.location === 'mongo') { - const deployableVersionObj = await models.DeployableVersion.findOne({org_id, channel_id: channel_uuid, uuid: version_uuid }); - if (!deployableVersionObj) { - throw `DeployableVersion is not found for ${channel.name}:${channel.uuid}/${versionObj.name}:${versionObj.uuid}.`; - } + const deployableVersionObj = await models.DeployableVersion.findOne({org_id, channel_id: channel_uuid, uuid: version_uuid }); + if (!deployableVersionObj) { + throw `DeployableVersion is not found for ${channel.name}:${channel.uuid}/${versionObj.name}:${versionObj.uuid}.`; + } + + if (versionObj.location === 'mongo') { deployableVersionObj.content = await decryptOrgData(orgKey, deployableVersionObj.content); - return deployableVersionObj; - } else { - //TODO: implement for S3 - throw 'fix me, not implement for S3 yet'; } + else if(versionObj.location === 's3'){ + const url = deployableVersionObj.content; + const urlObj = new URL(url); + const fullPath = urlObj.pathname; + var parts = _.filter(_.split(fullPath, '/')); + var bucketName = parts.shift(); + var path = `${parts.join('/')}`; + + const s3Client = new S3ClientClass(conf); + deployableVersionObj.content = await s3Client.getAndDecryptFile(bucketName, path, orgKey, deployableVersionObj.iv); + } + else { + throw `versionObj.location="${versionObj.location}" not implemented yet`; + } + return deployableVersionObj; }catch(err){ logger.error(err, `${queryName} encountered an error when serving ${req_id}.`); throw err; @@ -173,46 +187,25 @@ const channelResolvers = { const iv = crypto.randomBytes(16); const ivText = iv.toString('base64'); - const location = 'mongo'; - - // todo: enable s3 - // let location, data; - // - // if (conf.s3.endpoint) { - // try { - // const resourceName = channel.name + '-' + version.name; - // const bucket = `${conf.s3.bucketPrefix}-${orgId.toLowerCase()}`; - // const s3Client = new S3ClientClass(conf); - // try { - // const exists = await s3Client.bucketExists(bucket); - // if (!exists) { - // logger.warn('bucket does not exist', { bucket }); - // await s3Client.createBucket(bucket); - // } - // } catch (error) { - // logger.error('could not create bucket', { bucket: bucket }); - // throw error; - // } - // const s3 = new AWS.S3(conf.s3); - // const key = Buffer.concat([Buffer.from(req.orgKey)], 32); - // const encrypt = crypto.createCipheriv(algorithm, key, iv); - // const pipe = req.pipe(encrypt); - // const params = {Bucket: bucket, Key: resourceName, Body: pipe}; - // const upload = s3.upload( params ); - // await upload.promise(); - // - // data = `https://${conf.s3.endpoint}/${bucket}/${resourceName}`; - // location = 's3'; - // } catch (error) { - // logger.error( 'S3 upload error', error ); - // throw error; - // } - // } else { - // data = await encryptResource(req); - // location = 'mongo'; - // } - - const data = await encryptOrgData(orgKey, content); + let location = 'mongo'; + let data = await encryptOrgData(orgKey, content); + + if(conf.s3.endpoint){ + const resourceName = `${channel.name}-${name}`; + const bucketName = `${conf.s3.bucketPrefix}-${org_id.toLowerCase()}`; + + const s3Client = new S3ClientClass(conf); + + await s3Client.ensureBucketExists(bucketName); + + //data is now the s3 hostpath to the resource + const result = await s3Client.encryptAndUploadFile(bucketName, resourceName, content, orgKey, iv); + data = result.url; + + console.log(22222, result); + + location = 's3'; + } const deployableVersionObj = { _id: UUID(), diff --git a/app/conf.js b/app/conf.js index 819de3937..47d39ed30 100644 --- a/app/conf.js +++ b/app/conf.js @@ -25,7 +25,10 @@ const conf = { accessKeyId: process.env.S3_ACCESS_KEY_ID, secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, locationConstraint: process.env.S3_LOCATION_CONSTRAINT || 'us-standard', - bucketPrefix: process.env.S3_BUCKET_PREFIX || 'razee' + bucketPrefix: process.env.S3_BUCKET_PREFIX || 'razee', + s3ForcePathStyle: true, + signatureVersion: 'v4', + sslEnabled: !process.env.S3_DISABLE_SSL, //for local minio support } }; diff --git a/app/s3/s3Client.js b/app/s3/s3Client.js index 1b1ba3c0e..0b0b434b0 100644 --- a/app/s3/s3Client.js +++ b/app/s3/s3Client.js @@ -15,6 +15,10 @@ */ const clone = require('clone'); const AWS = require('aws-sdk'); +const crypto = require('crypto'); +const _ = require('lodash'); + +const encryptionAlgorithm = 'aes-256-cbc'; module.exports = class S3Client { constructor(options) { @@ -104,7 +108,8 @@ module.exports = class S3Client { const nop = { error: () => {}, info: () => {}, - debug: () => {} + debug: () => {}, + warn: () => {}, }; const result = this._log || nop; return result; @@ -113,4 +118,148 @@ module.exports = class S3Client { set log(logger) { this._log = logger; } + + async ensureBucketExists(bucketName){ + try { + const exists = await this.bucketExists(bucketName); + if(!exists){ + this.log.warn('bucket does not exist', { bucketName }); + await this.createBucket(bucketName); + } + }catch(err){ + this.log.error('could not create bucket', { bucketName }); + throw err; + } + } + + // async uploadFile(bucketName, path, content){ + // try { + // const exists = await this.bucketExists(bucketName); + // if(!exists){ + // this.log.warn('bucket does not exist', { bucketName }); + // await this.createBucket(bucketName); + // } + // }catch(err){ + // this.log.error('could not create bucket', { bucketName }); + // throw err; + // } + // + // const result = await this._aws.upload({ + // Bucket: bucketName, + // Key: path, + // Body: content, + // }).promise(); + // + // return `${this._conf.endpoint.match(/^http/i) ? 'https://' : ''}${this._conf.endpoint}/${bucketName}/${path}`; + // } + + // const s3 = new AWS.S3(conf.s3); + // const key = Buffer.concat([Buffer.from(req.orgKey)], 32); + // const encrypt = crypto.createCipheriv(algorithm, key, iv); + // const pipe = req.pipe(encrypt); + // const params = {Bucket: bucket, Key: resourceName, Body: pipe}; + // const upload = s3.upload( params ); + // await upload.promise(); + + async encryptAndUploadFile(bucketName, path, content, encryptionKey, iv=null){ + try { + const exists = await this.bucketExists(bucketName); + if(!exists){ + this.log.warn('bucket does not exist', { bucketName }); + await this.createBucket(bucketName); + } + }catch(err){ + this.log.error('could not create bucket', { bucketName }); + throw error; + } + + const key = Buffer.concat([Buffer.from(encryptionKey)], 32); + + if(!iv){ + iv = crypto.randomBytes(16); + } + const ivText = iv.toString('base64'); + + const cipher = crypto.createCipheriv(encryptionAlgorithm, key, iv); + + // let encrypted = cipher.update(content); + // encrypted = Buffer.concat([encrypted, cipher.final()]); + + const awsPromise = this._aws.upload({ + Bucket: bucketName, + Key: path, + Body: cipher, + }).promise(); + + cipher.write(content); + cipher.end(); + + await awsPromise; + + console.log('encrypted with', bucketName, path, content, key, iv, ivText); + + const url = `${this._conf.endpoint.match(/^http/i) ? '' : 'https://'}${this._conf.endpoint}/${bucketName}/${path}`; + + return { + url, ivText, + }; + } + + async getAndDecryptFile(bucketName, path, key, iv){ + console.log(1110, bucketName, path, key, iv); + if(_.isString(iv)){ + iv = Buffer.from(iv, 'base64'); + } + key = Buffer.concat([Buffer.from(key)], 32); + + var result = await this.getObject(bucketName, path).promise(); + var content = result.Body; + console.log(3333, bucketName, path, content, key, iv); + const decipher = crypto.createDecipheriv(encryptionAlgorithm, key, iv); + let out = decipher.update(content); + out = Buffer.concat([out, decipher.final()]); + out = out.toString('utf8'); + console.log(4444, out); + return out; + + + // try { + // const s3stream = this.getObject(bucketName, path).createReadStream(); + // const yaml = await readS3File(s3stream); + // return yaml; + // } + // catch (error) { + // this.log.error(error, 'Error retrieving data from s3 bucket'); + // throw(error); + // } + } }; + +setTimeout(async()=>{ + var s3Client = new module.exports(require('../conf.js').conf); + var bucketName = 'razee--k4tty77xnpmgjppfw'; + var path = 'blah'; + var content = 'this is teh content'; + var encryptionKey = 'orgApiKey-21fd8bfa-cc1d-43dd-988f-ddec98d72db7'; + var ivText = 'oRAApY8YmWQx5a98rUVkhg=='; + var iv = Buffer.from(ivText, 'base64'); + + console.log(11111, bucketName, path, content, encryptionKey, ivText, iv); + + await s3Client.encryptAndUploadFile(bucketName, path, content, encryptionKey, iv); + + console.log('uploaded'); + + await s3Client.getAndDecryptFile(bucketName, path, encryptionKey, iv); + + console.log('downloaded'); +},1); + +// const readS3File = async (readable) => { +// readable.setEncoding('utf8'); +// let data = ''; +// for await (const chunk of readable) { +// data += chunk; +// } +// return data; +// };