feat: add GCS as a FileStore

.env.example

@@ -34,6 +34,7 @@ GITLAB_CLIENT_SECRET='key_GITLAB_CLIENT_SECRET'
 GOOGLE_CLOUD_CLIENT_EMAIL='key_GOOGLE_CLOUD_CLIENT_EMAIL'
 GOOGLE_CLOUD_PRIVATE_KEY='key_GOOGLE_CLOUD_PRIVATE_KEY'
 GOOGLE_CLOUD_PRIVATE_KEY_ID='key_GOOGLE_CLOUD_PRIVATE_KEY_ID'
+GOOGLE_GCS_BUCKET=''
 GRAPHQL_HOST='localhost:3000'
 GRAPHQL_PROTOCOL='http'
 HOST='localhost'

package.json

@@ -70,7 +70,7 @@
  "caniuse-lite": "~1.0.0",
  "git-url-parse": "12.0.0",
  "fbjs": "^3.0.0",
- "undici": "^5.8.2",
+ "undici": "^5.23.0",
  "parse-url": "^8.1.0",
  "rethinkdb-ts": "2.5.1",
  "recursive-readdir": "^2.2.3",

packages/gql-executor/package.json

@@ -29,6 +29,6 @@
  "dd-trace": "^4.2.0",
  "parabol-client": "^6.123.1",
  "parabol-server": "^6.123.1",
- "undici": "^5.22.1"
+ "undici": "^5.23.0"
  }
 }

packages/server/fileStorage/FileStoreManager.ts

@@ -2,6 +2,8 @@ import generateUID from '../generateUID'
 
 export default abstract class FileStoreManager {
  abstract checkExists(fileName: string): Promise<boolean>
+ abstract prependPath(partialPath: string): string
+ abstract getPublicFileLocation(fullPath: string): string
 
  protected abstract putFile(file: Buffer, fullPath: string): Promise<string>
  protected abstract putUserFile(file: Buffer, partialPath: string): Promise<string>

packages/server/fileStorage/GCSManager.ts

@@ -0,0 +1,159 @@
+import {sign} from 'jsonwebtoken'
+import mime from 'mime-types'
+import path from 'path'
+import FileStoreManager from './FileStoreManager'
+
+interface CloudKey {
+ clientEmail: string
+ privateKeyId: string
+ privateKey: string
+}
+
+export default class GCSManager extends FileStoreManager {
+ static GOOGLE_EXPIRY = 3600
+ // e.g. development, production
+ private envSubDir: string
+ // e.g. action-files.parabol.co
+ private bucket: string
+ private accessToken: string | undefined
+
+ // The CDN_BASE_URL without the env, e.g. storage.google.com/:bucket
+ private baseUrl: string
+ private cloudKey: CloudKey
+ constructor() {
+ super()
+ const {
+ CDN_BASE_URL,
+ GOOGLE_GCS_BUCKET,
+ GOOGLE_CLOUD_CLIENT_EMAIL,
+ GOOGLE_CLOUD_PRIVATE_KEY,
+ GOOGLE_CLOUD_PRIVATE_KEY_ID
+ } = process.env
+ if (!CDN_BASE_URL || CDN_BASE_URL === 'key_CDN_BASE_URL') {
+ throw new Error('CDN_BASE_URL ENV VAR NOT SET')
+ }
+
+ if (!GOOGLE_CLOUD_CLIENT_EMAIL || !GOOGLE_CLOUD_PRIVATE_KEY_ID || !GOOGLE_CLOUD_PRIVATE_KEY) {
+ throw new Error(
+ 'Env Vars GOOGLE_CLOUD_CLIENT_EMAIL,GOOGLE_CLOUD_PRIVATE_ID,GOOGLE_CLOUD_PRIVATE_KEY must be set'
+ )
+ }
+
+ if (!GOOGLE_GCS_BUCKET) {
+ throw new Error('GOOGLE_GCS_BUCKET ENV VAR NOT SET')
+ }
+ const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
+ const {hostname, pathname} = baseUrl
+ if (!hostname || !pathname) {
+ throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
+ }
+ if (pathname.endsWith('/'))
+ throw new Error('CDN_BASE_URL must end with the env, no trailing slash, e.g. /production')
+
+ this.envSubDir = pathname.split('/').at(-1) as string
+
+ this.baseUrl = baseUrl.href.slice(0, baseUrl.href.lastIndexOf(this.envSubDir))
+
+ this.bucket = GOOGLE_GCS_BUCKET
+ this.cloudKey = {
+ clientEmail: GOOGLE_CLOUD_CLIENT_EMAIL,
+ privateKey: GOOGLE_CLOUD_PRIVATE_KEY.replace(/\\n/gm, '\n'),
+ privateKeyId: GOOGLE_CLOUD_PRIVATE_KEY_ID
+ }
+ // refresh the token every hour
+ // do this on an interval vs. on demand to reduce request latency
+ // unref it so things like pushToCDN can exit
+ setInterval(async () => {
+ this.accessToken = await this.getFreshAccessToken()
+ }, (GCSManager.GOOGLE_EXPIRY - 100) * 1000).unref()
+ }
+
+ private async getFreshAccessToken() {
+ const authUrl = 'https://www.googleapis.com/oauth2/v4/token'
+ const {clientEmail, privateKeyId, privateKey} = this.cloudKey
+ try {
+ // GCS only accepts OAuth2 Tokens
+ // To get a token, we self-sign a JWT, then trade it in for an OAuth2 Token
+ const jwt = sign(
+ {
+ scope: 'https://www.googleapis.com/auth/devstorage.read_write'
+ },
+ privateKey,
+ {
+ algorithm: 'RS256',
+ audience: authUrl,
+ subject: clientEmail,
+ issuer: clientEmail,
+ keyid: privateKeyId,
+ expiresIn: GCSManager.GOOGLE_EXPIRY
+ }
+ )
+ const accessTokenRes = await fetch(authUrl, {
+ method: 'POST',
+ body: JSON.stringify({
+ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+ assertion: jwt
+ })
+ })
+ const accessTokenJson = await accessTokenRes.json()
+ return accessTokenJson.access_token
+ } catch (e) {
+ return undefined
+ }
+ }
+
+ private async getAccessToken() {
+ if (this.accessToken) return this.accessToken
+ this.accessToken = await this.getFreshAccessToken()
+ return this.accessToken
+ }
+
+ protected async putUserFile(file: Buffer, partialPath: string) {
+ const fullPath = this.prependPath(partialPath)
+ return this.putFile(file, fullPath)
+ }
+
+ protected async putFile(file: Buffer, fullPath: string) {
+ const url = new URL(`https://storage.googleapis.com/upload/storage/v1/b/${this.bucket}/o`)
+ url.searchParams.append('uploadType', 'media')
+ url.searchParams.append('name', fullPath)
+ const accessToken = await this.getAccessToken()
+ try {
+ await fetch(url, {
+ method: 'POST',
+ body: file,
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ Accept: 'application/json',
+ 'Content-Type': mime.lookup(fullPath) || ''
+ }
+ })
+ } catch (e) {
+ // https://github.com/nodejs/undici/issues/583#issuecomment-1577475664
+ // GCS will cause undici to error randomly with `SocketError: other side closed` `code: 'UND_ERR_SOCKET'`
+ if ((e as any).cause?.code === 'UND_ERR_SOCKET') {
+ console.log(' Retrying GCS Post:', fullPath)
+ await this.putFile(file, fullPath)
+ }
+ }
+ return this.getPublicFileLocation(fullPath)
+ }
+
+ putBuildFile(file: Buffer, partialPath: string): Promise<string> {
+ const fullPath = path.join(this.envSubDir, 'build', partialPath)
+ return this.putFile(file, fullPath)
+ }
+
+ prependPath(partialPath: string) {
+ return path.join(this.envSubDir, 'store', partialPath)
+ }
+ getPublicFileLocation(fullPath: string) {
+ return encodeURI(`${this.baseUrl}${fullPath}`)
+ }
+ async checkExists(partialPath: string) {
+ const fullPath = encodeURIComponent(this.prependPath(partialPath))
+ const url = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o/${fullPath}`
+ const res = await fetch(url)
+ return res.status !== 404
+ }
+}

packages/server/fileStorage/LocalFileStoreManager.ts

@@ -11,13 +11,6 @@ export default class LocalFileSystemManager extends FileStoreManager {
  throw new Error('Env Vars PROTO and HOST must be set if FILE_STORE_PROVIDER=local')
  }
  }
- private prependPath(partialPath: string): string {
- return path.join('self-hosted', partialPath)
- }
-
- protected getPublicFileLocation(fullPath: string): string {
- return encodeURI(makeAppURL(appOrigin, fullPath))
- }
 
  protected async putUserFile(file: Buffer, partialPath: string) {
  const fullPath = this.prependPath(partialPath)
@@ -30,6 +23,14 @@ export default class LocalFileSystemManager extends FileStoreManager {
  return this.getPublicFileLocation(fullPath)
  }
 
+ prependPath(partialPath: string): string {
+ return path.join('self-hosted', partialPath)
+ }
+
+ getPublicFileLocation(fullPath: string): string {
+ return encodeURI(makeAppURL(appOrigin, fullPath))
+ }
+
  async putBuildFile() {
  console.error(
  'Cannot call `putBuildFile` when using Local File Storage. The build files are already there'

packages/server/fileStorage/S3FileStoreManager.ts

@@ -6,8 +6,11 @@ import FileStoreManager from './FileStoreManager'
 export default class S3Manager extends FileStoreManager {
  // e.g. development, production
  private envSubDir: string
- // e.g. action-files.parabol.co
+ // e.g. action-files.parabol.co. Usually matches CDN_BASE_URL for DNS reasons
  private bucket: string
+
+ // e.g. https://action-files.parabol.co
+ private baseUrl: string
  private s3: S3Client
  constructor() {
  super()
@@ -24,20 +27,18 @@ export default class S3Manager extends FileStoreManager {
  if (!hostname || !pathname) {
  throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
  }
+ if (pathname.endsWith('/'))
+ throw new Error('CDN_BASE_URL must end with the env, no trailing slash, e.g. /production')
+
+ this.envSubDir = pathname.split('/').at(-1) as string
+
+ this.baseUrl = baseUrl.href.slice(0, baseUrl.href.lastIndexOf(this.envSubDir))
 
- this.envSubDir = pathname.replace(/^\//, '')
  this.bucket = AWS_S3_BUCKET
  this.s3 = new S3Client({
  region: AWS_REGION
  })
  }
- private prependPath(partialPath: string) {
- return path.join(this.envSubDir, 'store', partialPath)
- }
-
- private getPublicFileLocation(fullPath: string) {
- return encodeURI(`https://${this.bucket}/${fullPath}`)
- }
 
  protected async putUserFile(file: Buffer, partialPath: string) {
  const fullPath = this.prependPath(partialPath)
@@ -54,6 +55,14 @@ export default class S3Manager extends FileStoreManager {
  return this.getPublicFileLocation(fullPath)
  }
 
+ prependPath(partialPath: string) {
+ return path.join(this.envSubDir, 'store', partialPath)
+ }
+
+ getPublicFileLocation(fullPath: string) {
+ return encodeURI(`${this.baseUrl}${fullPath}`)
+ }
+
  putBuildFile(file: Buffer, partialPath: string): Promise<string> {
  const fullPath = path.join(this.envSubDir, 'build', partialPath)
  return this.putFile(file, fullPath)

packages/server/fileStorage/getFileStoreManager.ts

@@ -1,10 +1,12 @@
 import FileStoreManager from './FileStoreManager'
 import LocalFileSystemManager from './LocalFileStoreManager'
 import S3Manager from './S3FileStoreManager'
+import GCSManager from './GCSManager'
 
 let fileStoreManager: FileStoreManager
 const managers = {
  s3: S3Manager,
+ gcs: GCSManager,
  local: LocalFileSystemManager
 }
 

packages/server/graphql/mutations/helpers/getTemplateIllustrationUrl.ts

@@ -1,22 +1,19 @@
+import getFileStoreManager from '../../../fileStorage/getFileStoreManager'
+
 // this can be removed after we sunset the old meeting template UI
+type CDN_TYPE = 'local' | 's3' | 'gcs'
+
 const getTemplateIllustrationUrl = (filename: string) => {
- const cdnType = process.env.FILE_STORE_PROVIDER
+ const cdnType = process.env.FILE_STORE_PROVIDER as CDN_TYPE | undefined
+ const manager = getFileStoreManager()
+
  const partialPath = `Organization/aGhostOrg/template/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
- const {AWS_S3_BUCKET, CDN_BASE_URL} = process.env
- if (!CDN_BASE_URL) throw new Error('Missing Env: CDN_BASE_URL')
- const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://'))
- const {hostname, pathname} = baseUrl
- if (!hostname || !pathname) {
- throw new Error('CDN_BASE_URL ENV VAR IS INVALID')
- }
- const environment = pathname.replace(/^\//, '')
- if (!AWS_S3_BUCKET) throw new Error('Missing Env: AWS_S3_BUCKET')
- return `https://${AWS_S3_BUCKET}/${environment}/store/${partialPath}`
+ } else {
+ const fullPath = manager.prependPath(partialPath)
+ return manager.getPublicFileLocation(fullPath)
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
 }
 
 export default getTemplateIllustrationUrl

packages/server/package.json

@@ -137,6 +137,6 @@
  "stripe": "^9.13.0",
  "tslib": "^2.4.0",
  "uWebSockets.js": "uNetworking/uWebSockets.js#v20.19.0",
- "undici": "^5.22.1"
+ "undici": "^5.23.0"
  }
 }

packages/server/postgres/migrations/1685721573097_addPrePostMortemTemplates.ts

@@ -983,13 +983,12 @@ const getTemplateIllustrationUrl = (filename: string) => {
  const partialPath = `Organization/aGhostOrg/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
+ } else {
  const {CDN_BASE_URL} = process.env
  if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
  const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
  return `https://${hostPath}/store/${partialPath}`
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
 }
 
 const makeTemplate = (template: Template) => ({

packages/server/postgres/migrations/1686226641890_fixed-activities.ts

@@ -7,13 +7,12 @@ export async function up() {
  const partialPath = `Organization/aGhostOrg/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
+ } else {
  const {CDN_BASE_URL} = process.env
  if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
  const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
  return `https://${hostPath}/store/${partialPath}`
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
  }
 
  const client = new Client(getPgConfig())

packages/server/postgres/migrations/1690364653869_addOneOnOneMeetingType.ts

@@ -10,13 +10,12 @@ export async function up() {
  const partialPath = `Organization/aGhostOrg/template/${filename}`
  if (cdnType === 'local') {
  return `/self-hosted/${partialPath}`
- } else if (cdnType === 's3') {
+ } else {
  const {CDN_BASE_URL} = process.env
  if (!CDN_BASE_URL) throw new Error('Missng Env: CDN_BASE_URL')
  const hostPath = CDN_BASE_URL.replace(/^\/+/, '')
  return `https://${hostPath}/store/${partialPath}`
  }
- throw new Error('Mssing Env: FILE_STORE_PROVIDER')
  }
 
  const oneOnOneActivity = {

yarn.lock

@@ -21272,12 +21272,12 @@ unc-path-regex@^0.1.2:
  resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa"
  integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo=
 
-undici@^5.22.1, undici@^5.8.2:
- version "5.25.4"
- resolved "https://registry.yarnpkg.com/undici/-/undici-5.25.4.tgz#7d8ef81d94f84cd384986271e5e5599b6dff4296"
- integrity sha512-450yJxT29qKMf3aoudzFpIciqpx6Pji3hEWaXqXmanbXF58LTAGCKxcJjxMXWu3iG+Mudgo3ZUfDB6YDFd/dAw==
+undici@^5.23.0:
+ version "5.23.0"
+ resolved "https://registry.yarnpkg.com/undici/-/undici-5.23.0.tgz#e7bdb0ed42cebe7b7aca87ced53e6eaafb8f8ca0"
+ integrity sha512-1D7w+fvRsqlQ9GscLBwcAJinqcZGHUKjbOmXdlE/v8BvEGXjeWAax+341q44EuTcHXXnfyKNbKRq4Lg7OzhMmg==
  dependencies:
- "@fastify/busboy" "^2.0.0"
+ busboy "^1.6.0"
 
 unicode-canonical-property-names-ecmascript@^2.0.0:
  version "2.0.0"