feat: add GCS as a FileStore

.env.example

@@ -33,6 +33,7 @@ GITLAB_CLIENT_SECRET='key_GITLAB_CLIENT_SECRET'
 GOOGLE_CLOUD_CLIENT_EMAIL='key_GOOGLE_CLOUD_CLIENT_EMAIL'
 GOOGLE_CLOUD_PRIVATE_KEY='key_GOOGLE_CLOUD_PRIVATE_KEY'
 GOOGLE_CLOUD_PRIVATE_KEY_ID='key_GOOGLE_CLOUD_PRIVATE_KEY_ID'
+GOOGLE_GCS_BUCKET=''
 GRAPHQL_HOST='localhost:3000'
 GRAPHQL_PROTOCOL='http'
 HOST='localhost'

packages/server/fileStorage/FileStoreManager.ts

@@ -2,6 +2,8 @@ import generateUID from '../generateUID'
 
 export default abstract class FileStoreManager {
  abstract checkExists(fileName: string): Promise<boolean>
+ abstract prependPath(partialPath: string): string
+ abstract getPublicFileLocation(fullPath: string): string
 
  protected abstract putFile(file: Buffer, fullPath: string): Promise<string>
  protected abstract putUserFile(file: Buffer, partialPath: string): Promise<string>

packages/server/fileStorage/GCSManager.ts

@@ -0,0 +1,150 @@
+import {sign} from 'jsonwebtoken'
+import mime from 'mime-types'
+import path from 'path'
+import FileStoreManager from './FileStoreManager'
+
+interface CloudKey {
+ clientEmail: string
+ privateKeyId: string
+ privateKey: string
+}
+
+export default class GCSManager extends FileStoreManager {
+ static GOOGLE_EXPIRY = 3600
+ // e.g. development, production
+ private envSubDir: string
+ // e.g. action-files.parabol.co
+ private bucket: string
+ private accessToken: string | undefined
+ private baseUrl: string
+ private cloudKey: CloudKey
+ constructor() {
+ super()
+ const {
+ CDN_BASE_URL,
+ GOOGLE_GCS_BUCKET,
+ GOOGLE_CLOUD_CLIENT_EMAIL,
+ GOOGLE_CLOUD_PRIVATE_KEY,
+ GOOGLE_CLOUD_PRIVATE_KEY_ID
+ } = process.env
+ if (!CDN_BASE_URL || CDN_BASE_URL === 'key_CDN_BASE_URL') {
+ throw new Error('CDN_BASE_URL ENV VAR NOT SET')
+ }
+
+ if (!GOOGLE_CLOUD_CLIENT_EMAIL || !GOOGLE_CLOUD_PRIVATE_KEY_ID || !GOOGLE_CLOUD_PRIVATE_KEY) {
+ throw new Error(
+ 'Env Vars GOOGLE_CLOUD_CLIENT_EMAIL,GOOGLE_CLOUD_PRIVATE_ID,GOOGLE_CLOUD_PRIVATE_KEY must be set'
+ )
+ }
+
+ if (!GOOGLE_GCS_BUCKET) {
+ throw new Error('GOOGLE_GCS_BUCKET ENV VAR NOT SET')
+ }
+ const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
+ const {hostname, pathname} = baseUrl
+ if (!hostname || !pathname) {
+ throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
+ }
+ this.baseUrl = baseUrl.href
+
+ this.envSubDir = pathname.replace(/^\//, '')
+ if (!this.envSubDir) {
+ throw new Error('CDN_BASE_URL must end with a pathname, e.g. /production')
+ }
+ if (this.envSubDir.endsWith('/')) {
+ throw new Error('CDN_BASE_URL must not end with a trailing slash /')
+ }
+
+ this.bucket = GOOGLE_GCS_BUCKET
+ this.cloudKey = {
+ clientEmail: GOOGLE_CLOUD_CLIENT_EMAIL,
+ privateKey: GOOGLE_CLOUD_PRIVATE_KEY.replace(/\\n/gm, '\n'),
+ privateKeyId: GOOGLE_CLOUD_PRIVATE_KEY_ID
+ }
+ // refresh the token every hour
+ // do this on an interval vs. on demand to reduce request latency
+ setInterval(async () => {
+ this.accessToken = await this.getFreshAccessToken()
+ }, (GCSManager.GOOGLE_EXPIRY - 100) * 1000)
+ }
+
+ private async getFreshAccessToken() {
+ const authUrl = 'https://www.googleapis.com/oauth2/v4/token'
+ const {clientEmail, privateKeyId, privateKey} = this.cloudKey
+ try {
+ // GCS only accepts OAuth2 Tokens
+ // To get a token, we self-sign a JWT, then trade it in for an OAuth2 Token
+ const jwt = sign(
+ {
+ scope: 'https://www.googleapis.com/auth/devstorage.read_write'
+ },
+ privateKey,
+ {
+ algorithm: 'RS256',
+ audience: authUrl,
+ subject: clientEmail,
+ issuer: clientEmail,
+ keyid: privateKeyId,
+ expiresIn: GCSManager.GOOGLE_EXPIRY
+ }
+ )
+ const accessTokenRes = await fetch(authUrl, {
+ method: 'POST',
+ body: JSON.stringify({
+ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+ assertion: jwt
+ })
+ })
+ const accessTokenJson = await accessTokenRes.json()
+ return accessTokenJson.access_token
+ } catch (e) {
+ return undefined
+ }
+ }
+
+ private async getAccessToken() {
+ if (this.accessToken) return this.accessToken
+ this.accessToken = await this.getFreshAccessToken()
+ return this.accessToken
+ }
+
+ protected async putUserFile(file: Buffer, partialPath: string) {
+ const fullPath = this.prependPath(partialPath)
+ return this.putFile(file, fullPath)
+ }
+
+ protected async putFile(file: Buffer, fullPath: string) {
+ const url = new URL(`https://storage.googleapis.com/upload/storage/v1/b/${this.bucket}/o`)
+ url.searchParams.append('uploadType', 'media')
+ url.searchParams.append('name', fullPath)
+ const accessToken = await this.getAccessToken()
+ await fetch(url, {
+ method: 'POST',
+ body: file,
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ Accept: 'application/json',
+ 'Content-Type': mime.lookup(fullPath) || ''
+ }
+ })
+ return this.getPublicFileLocation(fullPath)
+ }
+
+ putBuildFile(file: Buffer, partialPath: string): Promise<string> {
+ const fullPath = path.join(this.envSubDir, 'build', partialPath)
+ return this.putFile(file, fullPath)
+ }
+
+ prependPath(partialPath: string) {
+ return path.join(this.envSubDir, 'store', partialPath)
+ }
+ getPublicFileLocation(fullPath: string) {
+ return encodeURI(`https://${this.baseUrl}${fullPath}`)
+ }
+ async checkExists(partialPath: string) {
+ const fullPath = encodeURIComponent(this.prependPath(partialPath))
+ const url = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o/${fullPath}`
+ const res = await fetch(url)
+ return res.status !== 404
+ }
+}

packages/server/fileStorage/LocalFileStoreManager.ts

@@ -11,13 +11,6 @@ export default class LocalFileSystemManager extends FileStoreManager {
  throw new Error('Env Vars PROTO and HOST must be set if FILE_STORE_PROVIDER=local')
  }
  }
- private prependPath(partialPath: string): string {
- return path.join('self-hosted', partialPath)
- }
-
- protected getPublicFileLocation(fullPath: string): string {
- return encodeURI(makeAppURL(appOrigin, fullPath))
- }
 
  protected async putUserFile(file: Buffer, partialPath: string) {
  const fullPath = this.prependPath(partialPath)
@@ -30,6 +23,14 @@ export default class LocalFileSystemManager extends FileStoreManager {
  return this.getPublicFileLocation(fullPath)
  }
 
+ prependPath(partialPath: string): string {
+ return path.join('self-hosted', partialPath)
+ }
+
+ getPublicFileLocation(fullPath: string): string {
+ return encodeURI(makeAppURL(appOrigin, fullPath))
+ }
+
  async putBuildFile() {
  console.error(
  'Cannot call `putBuildFile` when using Local File Storage. The build files are already there'

packages/server/fileStorage/S3FileStoreManager.ts

@@ -6,8 +6,11 @@ import FileStoreManager from './FileStoreManager'
 export default class S3Manager extends FileStoreManager {
  // e.g. development, production
  private envSubDir: string
- // e.g. action-files.parabol.co
+ // e.g. action-files.parabol.co. Usually matches CDN_BASE_URL for DNS reasons
  private bucket: string
+
+ // e.g. https://action-files.parabol.co
+ private baseUrl: string
  private s3: S3Client
  constructor() {
  super()
@@ -24,20 +27,19 @@ export default class S3Manager extends FileStoreManager {
  if (!hostname || !pathname) {
  throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
  }
-
+ this.baseUrl = baseUrl.href
  this.envSubDir = pathname.replace(/^\//, '')
+ if (!this.envSubDir) {
+ throw new Error('CDN_BASE_URL must end with a pathname, e.g. /production')
+ }
+ if (this.envSubDir.endsWith('/')) {
+ throw new Error('CDN_BASE_URL must not end with a trailing slash /')
+ }
  this.bucket = AWS_S3_BUCKET
  this.s3 = new S3Client({
  region: AWS_REGION
  })
  }
- private prependPath(partialPath: string) {
- return path.join(this.envSubDir, 'store', partialPath)
- }
-
- private getPublicFileLocation(fullPath: string) {
- return encodeURI(`https://${this.bucket}/${fullPath}`)
- }
 
  protected async putUserFile(file: Buffer, partialPath: string) {
  const fullPath = this.prependPath(partialPath)
@@ -54,6 +56,14 @@ export default class S3Manager extends FileStoreManager {
  return this.getPublicFileLocation(fullPath)
  }
 
+ prependPath(partialPath: string) {
+ return path.join(this.envSubDir, 'store', partialPath)
+ }
+
+ getPublicFileLocation(fullPath: string) {
+ return encodeURI(`https://${this.baseUrl}${fullPath}`)
+ }
+
  putBuildFile(file: Buffer, partialPath: string): Promise<string> {
  const fullPath = path.join(this.envSubDir, 'build', partialPath)
  return this.putFile(file, fullPath)

packages/server/fileStorage/getFileStoreManager.ts

@@ -1,10 +1,12 @@
 import FileStoreManager from './FileStoreManager'
 import LocalFileSystemManager from './LocalFileStoreManager'
 import S3Manager from './S3FileStoreManager'
+import GCSManager from './GCSManager'
 
 let fileStoreManager: FileStoreManager
 const managers = {
  s3: S3Manager,
+ gcs: GCSManager,
  local: LocalFileSystemManager
 }
 

packages/server/graphql/mutations/helpers/getTemplateIllustrationUrl.ts

@@ -1,22 +1,19 @@
+import getFileStoreManager from '../../../fileStorage/getFileStoreManager'
+
 // this can be removed after we sunset the old meeting template UI
+type CDN_TYPE = 'local' | 's3' | 'gcs'
+
 const getTemplateIllustrationUrl = (filename: string) => {
- const cdnType = process.env.FILE_STORE_PROVIDER
+ const cdnType = process.env.FILE_STORE_PROVIDER as CDN_TYPE | undefined
+ const manager = getFileStoreManager()
+
  const partialPath = `Organization/aGhostOrg/template/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
- const {AWS_S3_BUCKET, CDN_BASE_URL} = process.env
- if (!CDN_BASE_URL) throw new Error('Missing Env: CDN_BASE_URL')
- const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
- const {hostname, pathname} = baseUrl
- if (!hostname || !pathname) {
- throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
- }
- const environment = pathname.replace(/^\//, '')
- if (!AWS_S3_BUCKET) throw new Error('Missing Env: AWS_S3_BUCKET')
- return `https://${AWS_S3_BUCKET}/${environment}/store/${partialPath}`
+ } else {
+ const fullPath = manager.prependPath(partialPath)
+ return manager.getPublicFileLocation(fullPath)
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
 }
 
 export default getTemplateIllustrationUrl

packages/server/postgres/migrations/1685721573097_addPrePostMortemTemplates.ts

@@ -982,13 +982,12 @@ const getTemplateIllustrationUrl = (filename: string) => {
  const partialPath = `Organization/aGhostOrg/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
+ } else {
  const {CDN_BASE_URL} = process.env
  if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
  const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
  return `https://${hostPath}/store/${partialPath}`
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
 }
 
 const makeTemplate = (template: Template) => ({

packages/server/postgres/migrations/1686226641890_fixed-activities.ts

@@ -7,13 +7,12 @@ export async function up() {
  const partialPath = `Organization/aGhostOrg/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
+ } else {
  const {CDN_BASE_URL} = process.env
  if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
  const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
  return `https://${hostPath}/store/${partialPath}`
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
  }
 
  const client = new Client(getPgConfig())

packages/server/postgres/migrations/1690364653869_addOneOnOneMeetingType.ts

@@ -10,13 +10,12 @@ export async function up() {
  const partialPath = `Organization/aGhostOrg/template/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
+ } else {
  const {CDN_BASE_URL} = process.env
  if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
  const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
  return `https://${hostPath}/store/${partialPath}`
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
  }
 
  const oneOnOneActivity = {