feat: Add S3 support for uploads (#938)

plankanban · Nov 11, 2024 · 950a070 · 950a070
1 parent f6ea10d
commit 950a070
Show file tree

Hide file tree

Showing 16 changed files with 579 additions and 24 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -80,6 +80,14 @@ services:
       # - TELEGRAM_BOT_TOKEN=
       # - TELEGRAM_CHAT_ID=
       # - TELEGRAM_THREAD_ID=
+
+      # Attachments S3
+      # - S3_ENABLE=true
+      # - S3_REGION=
+      # - S3_ENDPOINT=
+      # - S3_BUCKET=
+      # - S3_ACCESS_KEY=
+      # - S3_SECRET_KEY=
     depends_on:
       postgres:
         condition: service_healthy
@@ -93,7 +101,7 @@ services:
       - POSTGRES_DB=planka
       - POSTGRES_HOST_AUTH_METHOD=trust
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres -d planka"]
+      test: ['CMD-SHELL', 'pg_isready -U postgres -d planka']
       interval: 10s
       timeout: 5s
       retries: 5

diff --git a/server/.env.sample b/server/.env.sample
@@ -75,3 +75,10 @@ SECRET_KEY=notsecretkey
 ## Do not edit this
 
 TZ=UTC
+
+# S3_ENABLE=true
+# S3_REGION=
+# S3_ENDPOINT=
+# S3_BUCKET=
+# S3_ACCESS_KEY=
+# S3_SECRET_KEY=
diff --git a/server/api/helpers/attachments/delete-one.js b/server/api/helpers/attachments/delete-one.js
@@ -50,6 +50,24 @@ module.exports = {
     const attachment = await Attachment.archiveOne(inputs.record.id);
 
     if (attachment) {
+      try {
+        const type = attachment.type || 'local';
+        if (type === 's3') {
+          const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+          if (client) {
+            if (attachment.url) {
+              const parsedUrl = new URL(attachment.url);
+              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
+            }
+            if (attachment.thumb) {
+              const parsedUrl = new URL(attachment.thumb);
+              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
+            }
+          }
+        }
+      } catch (error) {
+        console.warn(error.stack); // eslint-disable-line no-console
+      }
       try {
         rimraf.sync(path.join(sails.config.custom.attachmentsPath, attachment.dirname));
       } catch (error) {

diff --git a/server/api/helpers/attachments/process-uploaded-file.js b/server/api/helpers/attachments/process-uploaded-file.js
@@ -23,6 +23,75 @@ module.exports = {
     const rootPath = path.join(sails.config.custom.attachmentsPath, dirname);
     const filePath = path.join(rootPath, filename);
 
+    if (sails.config.custom.s3Config) {
+      const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+      const s3Image = await client.upload({
+        Body: fs.createReadStream(inputs.file.fd),
+        Key: `attachments/${dirname}/${filename}`,
+        ContentType: inputs.file.type,
+      });
+
+      let image = sharp(inputs.file.fd, {
+        animated: true,
+      });
+
+      let metadata;
+      try {
+        metadata = await image.metadata();
+      } catch (error) {} // eslint-disable-line no-empty
+
+      const fileData = {
+        type: 's3',
+        dirname,
+        filename,
+        thumb: null,
+        image: null,
+        url: s3Image.Location,
+        name: inputs.file.filename,
+      };
+
+      if (metadata && !['svg', 'pdf'].includes(metadata.format)) {
+        let { width, pageHeight: height = metadata.height } = metadata;
+        if (metadata.orientation && metadata.orientation > 4) {
+          [image, width, height] = [image.rotate(), height, width];
+        }
+
+        const isPortrait = height > width;
+        const thumbnailsExtension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
+
+        try {
+          const resizeBuffer = await image
+            .resize(
+              256,
+              isPortrait ? 320 : undefined,
+              width < 256 || (isPortrait && height < 320)
+                ? {
+                    kernel: sharp.kernel.nearest,
+                  }
+                : undefined,
+            )
+            .toBuffer();
+          const s3Thumb = await client.upload({
+            Key: `attachments/${dirname}/thumbnails/cover-256.${thumbnailsExtension}`,
+            Body: resizeBuffer,
+            ContentType: inputs.file.type,
+          });
+          fileData.thumb = s3Thumb.Location;
+          fileData.image = { width, height };
+        } catch (error1) {
+          console.warn(error2.stack); // eslint-disable-line no-console
+        }
+      }
+
+      try {
+        rimraf.sync(inputs.file.fd);
+      } catch (error) {
+        console.warn(error.stack); // eslint-disable-line no-console
+      }
+
+      return fileData;
+    }
+
     fs.mkdirSync(rootPath);
     await moveFile(inputs.file.fd, filePath);
 
@@ -36,6 +105,7 @@ module.exports = {
     } catch (error) {} // eslint-disable-line no-empty
 
     const fileData = {
+      type: 'local',
       dirname,
       filename,
       image: null,

diff --git a/server/api/helpers/projects/process-uploaded-background-image-file.js b/server/api/helpers/projects/process-uploaded-background-image-file.js
@@ -33,9 +33,6 @@ module.exports = {
     }
 
     const dirname = uuid();
-    const rootPath = path.join(sails.config.custom.projectBackgroundImagesPath, dirname);
-
-    fs.mkdirSync(rootPath);
 
     let { width, pageHeight: height = metadata.height } = metadata;
     if (metadata.orientation && metadata.orientation > 4) {
@@ -44,6 +41,64 @@ module.exports = {
 
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
+    if (sails.config.custom.s3Config) {
+      const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+      let originalUrl = '';
+      let thumbUrl = '';
+
+      try {
+        const s3Original = await client.upload({
+          Body: await image.toBuffer(),
+          Key: `project-background-images/${dirname}/original.${extension}`,
+          ContentType: inputs.file.type,
+        });
+        originalUrl = s3Original.Location;
+
+        const resizeBuffer = await image
+          .resize(
+            336,
+            200,
+            width < 336 || height < 200
+              ? {
+                  kernel: sharp.kernel.nearest,
+                }
+              : undefined,
+          )
+          .toBuffer();
+        const s3Thumb = await client.upload({
+          Body: resizeBuffer,
+          Key: `project-background-images/${dirname}/cover-336.${extension}`,
+          ContentType: inputs.file.type,
+        });
+        thumbUrl = s3Thumb.Location;
+      } catch (error1) {
+        try {
+          client.delete({ Key: `project-background-images/${dirname}/original.${extension}` });
+        } catch (error2) {
+          console.warn(error2.stack); // eslint-disable-line no-console
+        }
+
+        throw 'fileIsNotImage';
+      }
+
+      try {
+        rimraf.sync(inputs.file.fd);
+      } catch (error) {
+        console.warn(error.stack); // eslint-disable-line no-console
+      }
+
+      return {
+        dirname,
+        extension,
+        original: originalUrl,
+        thumb: thumbUrl,
+      };
+    }
+
+    const rootPath = path.join(sails.config.custom.projectBackgroundImagesPath, dirname);
+
+    fs.mkdirSync(rootPath);
+
     try {
       await image.toFile(path.join(rootPath, `original.${extension}`));
 

diff --git a/server/api/helpers/projects/update-one.js b/server/api/helpers/projects/update-one.js
@@ -86,6 +86,21 @@ module.exports = {
         (!project.backgroundImage ||
           project.backgroundImage.dirname !== inputs.record.backgroundImage.dirname)
       ) {
+        try {
+          if (sails.config.custom.s3Config) {
+            const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+            if (client && inputs.record.backgroundImage && inputs.record.backgroundImage.original) {
+              const parsedUrl = new URL(inputs.record.backgroundImage.original);
+              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
+            }
+            if (client && inputs.record.backgroundImage && inputs.record.backgroundImage.thumb) {
+              const parsedUrl = new URL(inputs.record.backgroundImage.thumb);
+              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
+            }
+          }
+        } catch (error) {
+          console.warn(error.stack); // eslint-disable-line no-console
+        }
         try {
           rimraf.sync(
             path.join(

diff --git a/server/api/helpers/users/process-uploaded-avatar-file.js b/server/api/helpers/users/process-uploaded-avatar-file.js
@@ -33,9 +33,6 @@ module.exports = {
     }
 
     const dirname = uuid();
-    const rootPath = path.join(sails.config.custom.userAvatarsPath, dirname);
-
-    fs.mkdirSync(rootPath);
 
     let { width, pageHeight: height = metadata.height } = metadata;
     if (metadata.orientation && metadata.orientation > 4) {
@@ -44,6 +41,64 @@ module.exports = {
 
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
+    if (sails.config.custom.s3Config) {
+      const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+      let originalUrl = '';
+      let squareUrl = '';
+
+      try {
+        const s3Original = await client.upload({
+          Body: await image.toBuffer(),
+          Key: `user-avatars/${dirname}/original.${extension}`,
+          ContentType: inputs.file.type,
+        });
+        originalUrl = s3Original.Location;
+
+        const resizeBuffer = await image
+          .resize(
+            100,
+            100,
+            width < 100 || height < 100
+              ? {
+                  kernel: sharp.kernel.nearest,
+                }
+              : undefined,
+          )
+          .toBuffer();
+        const s3Square = await client.upload({
+          Body: resizeBuffer,
+          Key: `user-avatars/${dirname}/square-100.${extension}`,
+          ContentType: inputs.file.type,
+        });
+        squareUrl = s3Square.Location;
+      } catch (error1) {
+        try {
+          client.delete({ Key: `user-avatars/${dirname}/original.${extension}` });
+        } catch (error2) {
+          console.warn(error2.stack); // eslint-disable-line no-console
+        }
+
+        throw 'fileIsNotImage';
+      }
+
+      try {
+        rimraf.sync(inputs.file.fd);
+      } catch (error) {
+        console.warn(error.stack); // eslint-disable-line no-console
+      }
+
+      return {
+        dirname,
+        extension,
+        original: originalUrl,
+        square: squareUrl,
+      };
+    }
+
+    const rootPath = path.join(sails.config.custom.userAvatarsPath, dirname);
+
+    fs.mkdirSync(rootPath);
+
     try {
       await image.toFile(path.join(rootPath, `original.${extension}`));
 

diff --git a/server/api/helpers/users/update-one.js b/server/api/helpers/users/update-one.js
@@ -101,6 +101,21 @@ module.exports = {
         inputs.record.avatar &&
         (!user.avatar || user.avatar.dirname !== inputs.record.avatar.dirname)
       ) {
+        try {
+          if (sails.config.custom.s3Config) {
+            const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+            if (client && inputs.record.avatar && inputs.record.avatar.original) {
+              const parsedUrl = new URL(inputs.record.avatar.original);
+              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
+            }
+            if (client && inputs.record.avatar && inputs.record.avatar.square) {
+              const parsedUrl = new URL(inputs.record.avatar.square);
+              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
+            }
+          }
+        } catch (error) {
+          console.warn(error.stack); // eslint-disable-line no-console
+        }
         try {
           rimraf.sync(path.join(sails.config.custom.userAvatarsPath, inputs.record.avatar.dirname));
         } catch (error) {

diff --git a/server/api/helpers/utils/get-simple-storage-service-client.js b/server/api/helpers/utils/get-simple-storage-service-client.js
@@ -0,0 +1,45 @@
+const AWS = require('aws-sdk');
+
+class S3Client {
+  constructor(options) {
+    AWS.config.update({
+      accessKeyId: options.accessKeyId,
+      secretAccessKey: options.secretAccessKey,
+      region: options.region,
+    });
+    this.bucket = options.bucket;
+    this.client = new AWS.S3({
+      endpoint: options.endpoint,
+    });
+  }
+
+  upload({ Key, Body, ContentType }) {
+    return this.client
+      .upload({
+        Bucket: this.bucket,
+        Key,
+        Body,
+        ContentType,
+        ACL: 'public-read',
+      })
+      .promise();
+  }
+
+  delete({ Key }) {
+    return this.client
+      .deleteObject({
+        Bucket: this.bucket,
+        Key,
+      })
+      .promise();
+  }
+}
+
+module.exports = {
+  fn() {
+    if (sails.config.custom.s3Config) {
+      return new S3Client(sails.config.custom.s3Config);
+    }
+    return null;
+  },
+};