feat: add GCS as a FileStore (#8493)

* feat: add GCS as a FileStore Signed-off-by: Matt Krick <matt.krick@gmail.com> * self-review: fix comments Signed-off-by: Matt Krick <matt.krick@gmail.com> * fix: checkExists Signed-off-by: Matt Krick <matt.krick@gmail.com> * dockerize testing the branch * add recent filestore changes to gcs Signed-off-by: Matt Krick <matt.krick@gmail.com> * throw on trailing slash Signed-off-by: Matt Krick <matt.krick@gmail.com> * retry GCS flakey pushes Signed-off-by: Matt Krick <matt.krick@gmail.com> * dockerize Signed-off-by: Matt Krick <matt.krick@gmail.com> * bump yarn.lock Signed-off-by: Matt Krick <matt.krick@gmail.com> * fix yarn Signed-off-by: Matt Krick <matt.krick@gmail.com> --------- Signed-off-by: Matt Krick <matt.krick@gmail.com> Co-authored-by: Rafael Romero <rafael@parabol.co>
ParabolInc · Nov 13, 2023 · 9c33025 · 9c33025
1 parent 267dd17
commit 9c33025
Show file tree

Hide file tree

Showing 10 changed files with 203 additions and 35 deletions.
diff --git a/.env.example b/.env.example
@@ -34,6 +34,7 @@ GITLAB_CLIENT_SECRET='key_GITLAB_CLIENT_SECRET'
 GOOGLE_CLOUD_CLIENT_EMAIL='key_GOOGLE_CLOUD_CLIENT_EMAIL'
 GOOGLE_CLOUD_PRIVATE_KEY='key_GOOGLE_CLOUD_PRIVATE_KEY'
 GOOGLE_CLOUD_PRIVATE_KEY_ID='key_GOOGLE_CLOUD_PRIVATE_KEY_ID'
+GOOGLE_GCS_BUCKET=''
 GRAPHQL_HOST='localhost:3000'
 GRAPHQL_PROTOCOL='http'
 HOST='localhost'

diff --git a/packages/server/fileStorage/FileStoreManager.ts b/packages/server/fileStorage/FileStoreManager.ts
@@ -2,6 +2,8 @@ import generateUID from '../generateUID'
 
 export default abstract class FileStoreManager {
   abstract checkExists(fileName: string): Promise<boolean>
+  abstract prependPath(partialPath: string): string
+  abstract getPublicFileLocation(fullPath: string): string
 
   protected abstract putFile(file: Buffer, fullPath: string): Promise<string>
   protected abstract putUserFile(file: Buffer, partialPath: string): Promise<string>

diff --git a/packages/server/fileStorage/GCSManager.ts b/packages/server/fileStorage/GCSManager.ts
@@ -0,0 +1,159 @@
+import {sign} from 'jsonwebtoken'
+import mime from 'mime-types'
+import path from 'path'
+import FileStoreManager from './FileStoreManager'
+
+interface CloudKey {
+  clientEmail: string
+  privateKeyId: string
+  privateKey: string
+}
+
+export default class GCSManager extends FileStoreManager {
+  static GOOGLE_EXPIRY = 3600
+  // e.g. development, production
+  private envSubDir: string
+  // e.g. action-files.parabol.co
+  private bucket: string
+  private accessToken: string | undefined
+
+  // The CDN_BASE_URL without the env, e.g. storage.google.com/:bucket
+  private baseUrl: string
+  private cloudKey: CloudKey
+  constructor() {
+    super()
+    const {
+      CDN_BASE_URL,
+      GOOGLE_GCS_BUCKET,
+      GOOGLE_CLOUD_CLIENT_EMAIL,
+      GOOGLE_CLOUD_PRIVATE_KEY,
+      GOOGLE_CLOUD_PRIVATE_KEY_ID
+    } = process.env
+    if (!CDN_BASE_URL || CDN_BASE_URL === 'key_CDN_BASE_URL') {
+      throw new Error('CDN_BASE_URL ENV VAR NOT SET')
+    }
+
+    if (!GOOGLE_CLOUD_CLIENT_EMAIL || !GOOGLE_CLOUD_PRIVATE_KEY_ID || !GOOGLE_CLOUD_PRIVATE_KEY) {
+      throw new Error(
+        'Env Vars GOOGLE_CLOUD_CLIENT_EMAIL,GOOGLE_CLOUD_PRIVATE_ID,GOOGLE_CLOUD_PRIVATE_KEY must be set'
+      )
+    }
+
+    if (!GOOGLE_GCS_BUCKET) {
+      throw new Error('GOOGLE_GCS_BUCKET ENV VAR NOT SET')
+    }
+    const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
+    const {hostname, pathname} = baseUrl
+    if (!hostname || !pathname) {
+      throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
+    }
+    if (pathname.endsWith('/'))
+      throw new Error('CDN_BASE_URL must end with the env, no trailing slash, e.g. /production')
+
+    this.envSubDir = pathname.split('/').at(-1) as string
+
+    this.baseUrl = baseUrl.href.slice(0, baseUrl.href.lastIndexOf(this.envSubDir))
+
+    this.bucket = GOOGLE_GCS_BUCKET
+    this.cloudKey = {
+      clientEmail: GOOGLE_CLOUD_CLIENT_EMAIL,
+      privateKey: GOOGLE_CLOUD_PRIVATE_KEY.replace(/\\n/gm, '\n'),
+      privateKeyId: GOOGLE_CLOUD_PRIVATE_KEY_ID
+    }
+    // refresh the token every hour
+    // do this on an interval vs. on demand to reduce request latency
+    // unref it so things like pushToCDN can exit
+    setInterval(async () => {
+      this.accessToken = await this.getFreshAccessToken()
+    }, (GCSManager.GOOGLE_EXPIRY - 100) * 1000).unref()
+  }
+
+  private async getFreshAccessToken() {
+    const authUrl = 'https://www.googleapis.com/oauth2/v4/token'
+    const {clientEmail, privateKeyId, privateKey} = this.cloudKey
+    try {
+      // GCS only accepts OAuth2 Tokens
+      // To get a token, we self-sign a JWT, then trade it in for an OAuth2 Token
+      const jwt = sign(
+        {
+          scope: 'https://www.googleapis.com/auth/devstorage.read_write'
+        },
+        privateKey,
+        {
+          algorithm: 'RS256',
+          audience: authUrl,
+          subject: clientEmail,
+          issuer: clientEmail,
+          keyid: privateKeyId,
+          expiresIn: GCSManager.GOOGLE_EXPIRY
+        }
+      )
+      const accessTokenRes = await fetch(authUrl, {
+        method: 'POST',
+        body: JSON.stringify({
+          grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+          assertion: jwt
+        })
+      })
+      const accessTokenJson = await accessTokenRes.json()
+      return accessTokenJson.access_token
+    } catch (e) {
+      return undefined
+    }
+  }
+
+  private async getAccessToken() {
+    if (this.accessToken) return this.accessToken
+    this.accessToken = await this.getFreshAccessToken()
+    return this.accessToken
+  }
+
+  protected async putUserFile(file: Buffer, partialPath: string) {
+    const fullPath = this.prependPath(partialPath)
+    return this.putFile(file, fullPath)
+  }
+
+  protected async putFile(file: Buffer, fullPath: string) {
+    const url = new URL(`https://storage.googleapis.com/upload/storage/v1/b/${this.bucket}/o`)
+    url.searchParams.append('uploadType', 'media')
+    url.searchParams.append('name', fullPath)
+    const accessToken = await this.getAccessToken()
+    try {
+      await fetch(url, {
+        method: 'POST',
+        body: file,
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          Accept: 'application/json',
+          'Content-Type': mime.lookup(fullPath) || ''
+        }
+      })
+    } catch (e) {
+      // https://github.com/nodejs/undici/issues/583#issuecomment-1577475664
+      // GCS will cause undici to error randomly with `SocketError: other side closed` `code: 'UND_ERR_SOCKET'`
+      if ((e as any).cause?.code === 'UND_ERR_SOCKET') {
+        console.log('   Retrying GCS Post:', fullPath)
+        await this.putFile(file, fullPath)
+      }
+    }
+    return this.getPublicFileLocation(fullPath)
+  }
+
+  putBuildFile(file: Buffer, partialPath: string): Promise<string> {
+    const fullPath = path.join(this.envSubDir, 'build', partialPath)
+    return this.putFile(file, fullPath)
+  }
+
+  prependPath(partialPath: string) {
+    return path.join(this.envSubDir, 'store', partialPath)
+  }
+  getPublicFileLocation(fullPath: string) {
+    return encodeURI(`${this.baseUrl}${fullPath}`)
+  }
+  async checkExists(partialPath: string) {
+    const fullPath = encodeURIComponent(this.prependPath(partialPath))
+    const url = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o/${fullPath}`
+    const res = await fetch(url)
+    return res.status !== 404
+  }
+}
diff --git a/packages/server/fileStorage/LocalFileStoreManager.ts b/packages/server/fileStorage/LocalFileStoreManager.ts
@@ -11,13 +11,6 @@ export default class LocalFileSystemManager extends FileStoreManager {
       throw new Error('Env Vars PROTO and HOST must be set if FILE_STORE_PROVIDER=local')
     }
   }
-  private prependPath(partialPath: string): string {
-    return path.join('self-hosted', partialPath)
-  }
-
-  protected getPublicFileLocation(fullPath: string): string {
-    return encodeURI(makeAppURL(appOrigin, fullPath))
-  }
 
   protected async putUserFile(file: Buffer, partialPath: string) {
     const fullPath = this.prependPath(partialPath)
@@ -30,6 +23,14 @@ export default class LocalFileSystemManager extends FileStoreManager {
     return this.getPublicFileLocation(fullPath)
   }
 
+  prependPath(partialPath: string): string {
+    return path.join('self-hosted', partialPath)
+  }
+
+  getPublicFileLocation(fullPath: string): string {
+    return encodeURI(makeAppURL(appOrigin, fullPath))
+  }
+
   async putBuildFile() {
     console.error(
       'Cannot call `putBuildFile` when using Local File Storage. The build files are already there'

diff --git a/packages/server/fileStorage/S3FileStoreManager.ts b/packages/server/fileStorage/S3FileStoreManager.ts
@@ -6,8 +6,11 @@ import FileStoreManager from './FileStoreManager'
 export default class S3Manager extends FileStoreManager {
   // e.g. development, production
   private envSubDir: string
-  // e.g. action-files.parabol.co
+  // e.g. action-files.parabol.co. Usually matches CDN_BASE_URL for DNS reasons
   private bucket: string
+
+  // e.g. https://action-files.parabol.co
+  private baseUrl: string
   private s3: S3Client
   constructor() {
     super()
@@ -24,20 +27,18 @@ export default class S3Manager extends FileStoreManager {
     if (!hostname || !pathname) {
       throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
     }
+    if (pathname.endsWith('/'))
+      throw new Error('CDN_BASE_URL must end with the env, no trailing slash, e.g. /production')
+
+    this.envSubDir = pathname.split('/').at(-1) as string
+
+    this.baseUrl = baseUrl.href.slice(0, baseUrl.href.lastIndexOf(this.envSubDir))
 
-    this.envSubDir = pathname.replace(/^\//, '')
     this.bucket = AWS_S3_BUCKET
     this.s3 = new S3Client({
       region: AWS_REGION
     })
   }
-  private prependPath(partialPath: string) {
-    return path.join(this.envSubDir, 'store', partialPath)
-  }
-
-  private getPublicFileLocation(fullPath: string) {
-    return encodeURI(`https://${this.bucket}/${fullPath}`)
-  }
 
   protected async putUserFile(file: Buffer, partialPath: string) {
     const fullPath = this.prependPath(partialPath)
@@ -54,6 +55,14 @@ export default class S3Manager extends FileStoreManager {
     return this.getPublicFileLocation(fullPath)
   }
 
+  prependPath(partialPath: string) {
+    return path.join(this.envSubDir, 'store', partialPath)
+  }
+
+  getPublicFileLocation(fullPath: string) {
+    return encodeURI(`${this.baseUrl}${fullPath}`)
+  }
+
   putBuildFile(file: Buffer, partialPath: string): Promise<string> {
     const fullPath = path.join(this.envSubDir, 'build', partialPath)
     return this.putFile(file, fullPath)

diff --git a/packages/server/fileStorage/getFileStoreManager.ts b/packages/server/fileStorage/getFileStoreManager.ts
@@ -1,10 +1,12 @@
 import FileStoreManager from './FileStoreManager'
 import LocalFileSystemManager from './LocalFileStoreManager'
 import S3Manager from './S3FileStoreManager'
+import GCSManager from './GCSManager'
 
 let fileStoreManager: FileStoreManager
 const managers = {
   s3: S3Manager,
+  gcs: GCSManager,
   local: LocalFileSystemManager
 }
 

diff --git a/packages/server/graphql/mutations/helpers/getTemplateIllustrationUrl.ts b/packages/server/graphql/mutations/helpers/getTemplateIllustrationUrl.ts
@@ -1,22 +1,19 @@
+import getFileStoreManager from '../../../fileStorage/getFileStoreManager'
+
 // this can be removed after we sunset the old meeting template UI
+type CDN_TYPE = 'local' | 's3' | 'gcs'
+
 const getTemplateIllustrationUrl = (filename: string) => {
-  const cdnType = process.env.FILE_STORE_PROVIDER
+  const cdnType = process.env.FILE_STORE_PROVIDER as CDN_TYPE | undefined
+  const manager = getFileStoreManager()
+
   const partialPath = `Organization/aGhostOrg/template/${filename}`
   if (cdnType === 'local') {
     return `/self-hosted/${partialPath}`
-  } else if (cdnType === 's3') {
-    const {AWS_S3_BUCKET, CDN_BASE_URL} = process.env
-    if (!CDN_BASE_URL) throw new Error('Missing Env: CDN_BASE_URL')
-    const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
-    const {hostname, pathname} = baseUrl
-    if (!hostname || !pathname) {
-      throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
-    }
-    const environment = pathname.replace(/^\//, '')
-    if (!AWS_S3_BUCKET) throw new Error('Missing Env: AWS_S3_BUCKET')
-    return `https://${AWS_S3_BUCKET}/${environment}/store/${partialPath}`
+  } else {
+    const fullPath = manager.prependPath(partialPath)
+    return manager.getPublicFileLocation(fullPath)
   }
-  throw new Error('Mssing Env: FILE_STORE_PROVIDER')
 }
 
 export default getTemplateIllustrationUrl
diff --git a/packages/server/postgres/migrations/1685721573097_addPrePostMortemTemplates.ts b/packages/server/postgres/migrations/1685721573097_addPrePostMortemTemplates.ts
@@ -983,13 +983,12 @@ const getTemplateIllustrationUrl = (filename: string) => {
   const partialPath = `Organization/aGhostOrg/${filename}`
   if (cdnType === 'local') {
     return `/self-hosted/${partialPath}`
-  } else if (cdnType === 's3') {
+  } else {
     const {CDN_BASE_URL} = process.env
     if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
     const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
     return `https://${hostPath}/store/${partialPath}`
   }
-  throw new Error('Mssing Env: FILE_STORE_PROVIDER')
 }
 
 const makeTemplate = (template: Template) => ({

diff --git a/packages/server/postgres/migrations/1686226641890_fixed-activities.ts b/packages/server/postgres/migrations/1686226641890_fixed-activities.ts
@@ -7,13 +7,12 @@ export async function up() {
     const partialPath = `Organization/aGhostOrg/${filename}`
     if (cdnType === 'local') {
       return `/self-hosted/${partialPath}`
-    } else if (cdnType === 's3') {
+    } else {
       const {CDN_BASE_URL} = process.env
       if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
       const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
       return `https://${hostPath}/store/${partialPath}`
     }
-    throw new Error('Mssing Env: FILE_STORE_PROVIDER')
   }
 
   const client = new Client(getPgConfig())

diff --git a/packages/server/postgres/migrations/1690364653869_addOneOnOneMeetingType.ts b/packages/server/postgres/migrations/1690364653869_addOneOnOneMeetingType.ts
@@ -10,13 +10,12 @@ export async function up() {
     const partialPath = `Organization/aGhostOrg/template/${filename}`
     if (cdnType === 'local') {
       return `/self-hosted/${partialPath}`
-    } else if (cdnType === 's3') {
+    } else {
       const {CDN_BASE_URL} = process.env
       if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
       const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
       return `https://${hostPath}/store/${partialPath}`
     }
-    throw new Error('Mssing Env: FILE_STORE_PROVIDER')
   }
 
   const oneOnOneActivity = {