From 812e601c64948159afc2b3c4fdf9ae76372fa223 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Tue, 8 Oct 2024 11:24:05 -0400 Subject: [PATCH 01/13] don't remove, tag metadata --- services/app-web/src/s3/s3Amplify.ts | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 7712199f06..5c96da291b 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -78,9 +78,33 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { bucket: BucketShortName ): Promise => { try { - await Storage.remove(filename, { - bucket: bucketConfig[bucket], - }) + // Get the current metadata of the file + // Construct the full key including the bucket prefix + const fullKey = `${bucketConfig[bucket]}/${filename}` + + // Get the current metadata of the file + const { metadata } = await Storage.getProperties(fullKey) + + // Add or update the 'deleted' tag + const updatedMetadata = { + ...metadata, + deleted: 'true', + deletedAt: new Date().toISOString(), + } + + // Update the file's metadata + await Storage.copy( + { key: filename }, + { key: filename }, + { + bucket: bucketConfig[bucket], + metadata: updatedMetadata, + } + ) + + console.info( + `File ${filename} tagged as deleted in bucket ${bucket}` + ) return } catch (err) { assertIsS3PutError(err) From 70b2e400a91c378e729663418ff85a74170c8e51 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Tue, 8 Oct 2024 16:32:45 -0400 Subject: [PATCH 02/13] more debug --- services/app-web/src/s3/s3Amplify.ts | 41 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 5c96da291b..6bf3539e2c 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -77,15 +77,23 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { filename: string, bucket: BucketShortName ): Promise => { - try { - // Get the current metadata of the file - // Construct the full key including the bucket prefix - const fullKey = `${bucketConfig[bucket]}/${filename}` + // Construct the full key including the bucket prefix + const fullKey = `${bucketConfig[bucket]}/${filename}` + console.info(`Attempting to tag file as deleted: ${fullKey}`) + let metadata - // Get the current metadata of the file - const { metadata } = await Storage.getProperties(fullKey) + try { + const result = await Storage.getProperties(fullKey) + metadata = result.metadata + console.info('Successfully got file properties', metadata) + } catch (getPropertiesError) { + console.error('Error in getProperties:', getPropertiesError) + throw getPropertiesError + } + try { // Add or update the 'deleted' tag + console.info('Preparing updated metadata') const updatedMetadata = { ...metadata, deleted: 'true', @@ -93,20 +101,25 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { } // Update the file's metadata - await Storage.copy( - { key: filename }, - { key: filename }, - { - bucket: bucketConfig[bucket], - metadata: updatedMetadata, - } - ) + console.info('Copying file to update metadata') + try { + await Storage.copy( + { key: fullKey }, + { key: fullKey }, + { metadata: updatedMetadata } + ) + console.info('Successfully updated file metadata') + } catch (copyError) { + console.error('Error in Storage.copy:', copyError) + throw copyError + } console.info( `File ${filename} tagged as deleted in bucket ${bucket}` ) return } catch (err) { + console.error('Error in tagFileAsDeleted:', err) assertIsS3PutError(err) recordJSException(err) if (err.name === 'Error' && err.message === 'Network Error') { From a99225a94889b5a00f61ccde295ad03a553d4e52 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Tue, 8 Oct 2024 16:45:00 -0400 Subject: [PATCH 03/13] get metadata attributes --- services/uploads/serverless.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml index d672e72dde..90124ba4a1 100644 --- a/services/uploads/serverless.yml +++ b/services/uploads/serverless.yml @@ -26,6 +26,7 @@ provider: - Effect: 'Allow' Action: - s3:GetObject + - s3:GetObjectAttributes - s3:GetObjectTagging - s3:PutObject - s3:PutObjectAcl From d95a60b1771b0b223215d5eb5ac35a116eb9ad7d Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Tue, 8 Oct 2024 17:00:00 -0400 Subject: [PATCH 04/13] more permissions --- services/uploads/serverless.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml index 90124ba4a1..05310a798f 100644 --- a/services/uploads/serverless.yml +++ b/services/uploads/serverless.yml @@ -40,6 +40,14 @@ provider: - !Sub arn:aws:s3:::${self:service}-${sls:stage}-qa-${AWS::AccountId}/* - Effect: 'Allow' Action: + - s3:GetObject + - s3:GetObjectAttributes + - s3:GetObjectTagging + - s3:PutObject + - s3:PutObjectAcl + - s3:PutObjectTagging + - s3:PutObjectVersionTagging + - s3:DeleteObject - s3:ListBucket Resource: - !Sub arn:aws:s3:::${self:service}-${sls:stage}-uploads-${AWS::AccountId} From 830626f2cd1a37d18ab6c5b6096ee650da205dea Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 11:06:32 -0400 Subject: [PATCH 05/13] fix fullkey --- services/app-web/src/s3/s3Amplify.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 6bf3539e2c..713a380bed 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -78,7 +78,7 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { bucket: BucketShortName ): Promise => { // Construct the full key including the bucket prefix - const fullKey = `${bucketConfig[bucket]}/${filename}` + const fullKey = `${bucketConfig[bucket]}/allusers/${filename}` console.info(`Attempting to tag file as deleted: ${fullKey}`) let metadata From a39cee9876aed5acd606c9cad4785f753fac0cf0 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 11:18:41 -0400 Subject: [PATCH 06/13] debug --- services/app-web/src/s3/s3Amplify.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 713a380bed..204ff90300 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -83,7 +83,8 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { let metadata try { - const result = await Storage.getProperties(fullKey) + console.info(`filename: ${filename}`) + const result = await Storage.getProperties(filename) metadata = result.metadata console.info('Successfully got file properties', metadata) } catch (getPropertiesError) { From dc25c83e4433883f0eec0e7960ad145478a80edb Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 12:35:45 -0400 Subject: [PATCH 07/13] fix cognito perms --- services/ui-auth/serverless.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/ui-auth/serverless.yml b/services/ui-auth/serverless.yml index cf1bd6abd1..6bb502f7c0 100644 --- a/services/ui-auth/serverless.yml +++ b/services/ui-auth/serverless.yml @@ -211,9 +211,11 @@ resources: Action: - 's3:*' Resource: + - ${self:custom.document_uploads_bucket_arn} + - ${self:custom.document_uploads_bucket_arn}/* + - ${self:custom.qa_uploads_bucket_arn} + - ${self:custom.qa_uploads_bucket_arn}/* # Must use Join here. See: https://github.com/serverless/serverless/issues/3565 - - ${self:custom.document_uploads_bucket_arn}/allusers/* - - ${self:custom.qa_uploads_bucket_arn}/allusers/* # This private bucket is used by any files stored "private" by Amplify # https://docs.amplify.aws/lib/storage/configureaccess/q/platform/js/ - Fn::Join: From a056bbe261af8db4b3a4063c376dc9e41f87eace Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 12:43:28 -0400 Subject: [PATCH 08/13] use just the filename here --- services/app-web/src/s3/s3Amplify.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 204ff90300..425346d015 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -105,8 +105,8 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { console.info('Copying file to update metadata') try { await Storage.copy( - { key: fullKey }, - { key: fullKey }, + { key: filename }, + { key: filename }, { metadata: updatedMetadata } ) console.info('Successfully updated file metadata') From c1d14241d07f0c930a5a0a9e5595bd33c192eb5d Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 12:50:03 -0400 Subject: [PATCH 09/13] use a fullKey here --- services/app-web/src/s3/s3Amplify.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 425346d015..86b4b76a66 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -78,7 +78,7 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { bucket: BucketShortName ): Promise => { // Construct the full key including the bucket prefix - const fullKey = `${bucketConfig[bucket]}/allusers/${filename}` + const fullKey = `allusers/${filename}` console.info(`Attempting to tag file as deleted: ${fullKey}`) let metadata @@ -105,8 +105,8 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { console.info('Copying file to update metadata') try { await Storage.copy( - { key: filename }, - { key: filename }, + { key: fullKey }, + { key: fullKey }, { metadata: updatedMetadata } ) console.info('Successfully updated file metadata') From 3432312a25a2a682a9e426356ef2405796068879 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 17:48:03 -0400 Subject: [PATCH 10/13] just filename --- services/app-web/src/s3/s3Amplify.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 86b4b76a66..22b0168ef0 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -78,7 +78,9 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { bucket: BucketShortName ): Promise => { // Construct the full key including the bucket prefix - const fullKey = `allusers/${filename}` + const fullKey = filename.startsWith('allusers/') + ? filename + : `allusers/${filename}` console.info(`Attempting to tag file as deleted: ${fullKey}`) let metadata @@ -105,8 +107,8 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { console.info('Copying file to update metadata') try { await Storage.copy( - { key: fullKey }, - { key: fullKey }, + { key: filename }, + { key: filename }, { metadata: updatedMetadata } ) console.info('Successfully updated file metadata') From cb26a7602843cd49970c3139c9838167842fe363 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 18:00:51 -0400 Subject: [PATCH 11/13] cleanup --- services/app-web/src/s3/s3Amplify.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 22b0168ef0..6a789ac6db 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -77,18 +77,12 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { filename: string, bucket: BucketShortName ): Promise => { - // Construct the full key including the bucket prefix - const fullKey = filename.startsWith('allusers/') - ? filename - : `allusers/${filename}` - console.info(`Attempting to tag file as deleted: ${fullKey}`) + console.info(`Attempting to tag file as deleted: ${filename}`) let metadata try { - console.info(`filename: ${filename}`) const result = await Storage.getProperties(filename) metadata = result.metadata - console.info('Successfully got file properties', metadata) } catch (getPropertiesError) { console.error('Error in getProperties:', getPropertiesError) throw getPropertiesError @@ -96,7 +90,6 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { try { // Add or update the 'deleted' tag - console.info('Preparing updated metadata') const updatedMetadata = { ...metadata, deleted: 'true', @@ -104,14 +97,12 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { } // Update the file's metadata - console.info('Copying file to update metadata') try { await Storage.copy( { key: filename }, { key: filename }, { metadata: updatedMetadata } ) - console.info('Successfully updated file metadata') } catch (copyError) { console.error('Error in Storage.copy:', copyError) throw copyError @@ -122,7 +113,9 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { ) return } catch (err) { - console.error('Error in tagFileAsDeleted:', err) + console.error( + `Error in tagFileAsDeleted for ${filename}: ${err}` + ) assertIsS3PutError(err) recordJSException(err) if (err.name === 'Error' && err.message === 'Network Error') { @@ -132,7 +125,6 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { message: 'Error deleting file from the cloud.', } } - console.info('Unexpected Error deleting file from S3', err) throw err } }, From 7f5834a92849e174d79b5d9e447df8744a0ee60b Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 18:19:04 -0400 Subject: [PATCH 12/13] debug --- services/app-web/src/s3/s3Amplify.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 6a789ac6db..34af86eec5 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -95,22 +95,21 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { deleted: 'true', deletedAt: new Date().toISOString(), } + console.info(`updatedMetadata: ${updatedMetadata}`) // Update the file's metadata try { - await Storage.copy( + const res = await Storage.copy( { key: filename }, { key: filename }, { metadata: updatedMetadata } ) + console.info(`result: ${res}`) } catch (copyError) { console.error('Error in Storage.copy:', copyError) throw copyError } - console.info( - `File ${filename} tagged as deleted in bucket ${bucket}` - ) return } catch (err) { console.error( From 7286fca9e911620f5a2eceb5b2f2acb92d7138a0 Mon Sep 17 00:00:00 2001 From: Mojo Talantikite Date: Wed, 9 Oct 2024 18:40:28 -0400 Subject: [PATCH 13/13] debug --- services/app-web/src/s3/s3Amplify.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/app-web/src/s3/s3Amplify.ts b/services/app-web/src/s3/s3Amplify.ts index 34af86eec5..2c8a6bb13b 100644 --- a/services/app-web/src/s3/s3Amplify.ts +++ b/services/app-web/src/s3/s3Amplify.ts @@ -95,7 +95,9 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { deleted: 'true', deletedAt: new Date().toISOString(), } - console.info(`updatedMetadata: ${updatedMetadata}`) + console.info( + `updatedMetadata: ${JSON.stringify(updatedMetadata)}` + ) // Update the file's metadata try { @@ -104,7 +106,7 @@ function newAmplifyS3Client(bucketConfig: S3BucketConfigType): S3ClientT { { key: filename }, { metadata: updatedMetadata } ) - console.info(`result: ${res}`) + console.info(`result: ${JSON.stringify(res)}`) } catch (copyError) { console.error('Error in Storage.copy:', copyError) throw copyError