-
Notifications
You must be signed in to change notification settings - Fork 327
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: add GCS as a FileStore #8493
Merged
Changes from all commits
Commits
Show all changes
16 commits
Select commit
Hold shift + click to select a range
75f5a53
feat: add GCS as a FileStore
mattkrick 360f227
self-review: fix comments
mattkrick f41dcac
fix: checkExists
mattkrick aee560f
dockerize testing the branch
rafaelromcar-parabol 9182c18
Merge branch 'master' into feat/gcs
mattkrick 0eb9aae
Merge branch 'staging' into feat/gcs
mattkrick 95d3d6c
add recent filestore changes to gcs
mattkrick 6a509a4
Merge branch 'master' into feat/gcs
mattkrick 5778321
throw on trailing slash
mattkrick ba0c144
retry GCS flakey pushes
mattkrick e560327
Merge remote-tracking branch 'origin/master' into feat/gcs
mattkrick 33d59df
dockerize
mattkrick 59abb16
Merge branch 'master' into feat/gcs
mattkrick f4dfbf3
bump yarn.lock
mattkrick 68a332f
Merge branch 'master' into feat/gcs
mattkrick 70418a2
fix yarn
mattkrick File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,159 @@ | ||
import {sign} from 'jsonwebtoken' | ||
import mime from 'mime-types' | ||
import path from 'path' | ||
import FileStoreManager from './FileStoreManager' | ||
|
||
interface CloudKey { | ||
clientEmail: string | ||
privateKeyId: string | ||
privateKey: string | ||
} | ||
|
||
export default class GCSManager extends FileStoreManager { | ||
static GOOGLE_EXPIRY = 3600 | ||
// e.g. development, production | ||
private envSubDir: string | ||
// e.g. action-files.parabol.co | ||
private bucket: string | ||
private accessToken: string | undefined | ||
|
||
// The CDN_BASE_URL without the env, e.g. storage.google.com/:bucket | ||
private baseUrl: string | ||
private cloudKey: CloudKey | ||
constructor() { | ||
super() | ||
const { | ||
CDN_BASE_URL, | ||
GOOGLE_GCS_BUCKET, | ||
GOOGLE_CLOUD_CLIENT_EMAIL, | ||
GOOGLE_CLOUD_PRIVATE_KEY, | ||
GOOGLE_CLOUD_PRIVATE_KEY_ID | ||
} = process.env | ||
if (!CDN_BASE_URL || CDN_BASE_URL === 'key_CDN_BASE_URL') { | ||
throw new Error('CDN_BASE_URL ENV VAR NOT SET') | ||
} | ||
|
||
if (!GOOGLE_CLOUD_CLIENT_EMAIL || !GOOGLE_CLOUD_PRIVATE_KEY_ID || !GOOGLE_CLOUD_PRIVATE_KEY) { | ||
throw new Error( | ||
'Env Vars GOOGLE_CLOUD_CLIENT_EMAIL,GOOGLE_CLOUD_PRIVATE_ID,GOOGLE_CLOUD_PRIVATE_KEY must be set' | ||
) | ||
} | ||
|
||
if (!GOOGLE_GCS_BUCKET) { | ||
throw new Error('GOOGLE_GCS_BUCKET ENV VAR NOT SET') | ||
} | ||
const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://')) | ||
const {hostname, pathname} = baseUrl | ||
if (!hostname || !pathname) { | ||
throw new Error('CDN_BASE_URL ENV VAR IS INVALID') | ||
} | ||
if (pathname.endsWith('/')) | ||
throw new Error('CDN_BASE_URL must end with the env, no trailing slash, e.g. /production') | ||
|
||
this.envSubDir = pathname.split('/').at(-1) as string | ||
|
||
this.baseUrl = baseUrl.href.slice(0, baseUrl.href.lastIndexOf(this.envSubDir)) | ||
|
||
this.bucket = GOOGLE_GCS_BUCKET | ||
this.cloudKey = { | ||
clientEmail: GOOGLE_CLOUD_CLIENT_EMAIL, | ||
privateKey: GOOGLE_CLOUD_PRIVATE_KEY.replace(/\\n/gm, '\n'), | ||
privateKeyId: GOOGLE_CLOUD_PRIVATE_KEY_ID | ||
} | ||
// refresh the token every hour | ||
// do this on an interval vs. on demand to reduce request latency | ||
// unref it so things like pushToCDN can exit | ||
setInterval(async () => { | ||
this.accessToken = await this.getFreshAccessToken() | ||
}, (GCSManager.GOOGLE_EXPIRY - 100) * 1000).unref() | ||
} | ||
|
||
private async getFreshAccessToken() { | ||
const authUrl = 'https://www.googleapis.com/oauth2/v4/token' | ||
const {clientEmail, privateKeyId, privateKey} = this.cloudKey | ||
try { | ||
// GCS only accepts OAuth2 Tokens | ||
// To get a token, we self-sign a JWT, then trade it in for an OAuth2 Token | ||
const jwt = sign( | ||
{ | ||
scope: 'https://www.googleapis.com/auth/devstorage.read_write' | ||
}, | ||
privateKey, | ||
{ | ||
algorithm: 'RS256', | ||
audience: authUrl, | ||
subject: clientEmail, | ||
issuer: clientEmail, | ||
keyid: privateKeyId, | ||
expiresIn: GCSManager.GOOGLE_EXPIRY | ||
} | ||
) | ||
const accessTokenRes = await fetch(authUrl, { | ||
method: 'POST', | ||
body: JSON.stringify({ | ||
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', | ||
assertion: jwt | ||
}) | ||
}) | ||
const accessTokenJson = await accessTokenRes.json() | ||
return accessTokenJson.access_token | ||
} catch (e) { | ||
return undefined | ||
} | ||
} | ||
|
||
private async getAccessToken() { | ||
if (this.accessToken) return this.accessToken | ||
this.accessToken = await this.getFreshAccessToken() | ||
return this.accessToken | ||
} | ||
|
||
protected async putUserFile(file: Buffer, partialPath: string) { | ||
const fullPath = this.prependPath(partialPath) | ||
return this.putFile(file, fullPath) | ||
} | ||
|
||
protected async putFile(file: Buffer, fullPath: string) { | ||
const url = new URL(`https://storage.googleapis.com/upload/storage/v1/b/${this.bucket}/o`) | ||
url.searchParams.append('uploadType', 'media') | ||
url.searchParams.append('name', fullPath) | ||
const accessToken = await this.getAccessToken() | ||
try { | ||
await fetch(url, { | ||
method: 'POST', | ||
body: file, | ||
headers: { | ||
Authorization: `Bearer ${accessToken}`, | ||
Accept: 'application/json', | ||
'Content-Type': mime.lookup(fullPath) || '' | ||
} | ||
}) | ||
} catch (e) { | ||
// https://github.com/nodejs/undici/issues/583#issuecomment-1577475664 | ||
// GCS will cause undici to error randomly with `SocketError: other side closed` `code: 'UND_ERR_SOCKET'` | ||
if ((e as any).cause?.code === 'UND_ERR_SOCKET') { | ||
console.log(' Retrying GCS Post:', fullPath) | ||
await this.putFile(file, fullPath) | ||
} | ||
} | ||
return this.getPublicFileLocation(fullPath) | ||
} | ||
|
||
putBuildFile(file: Buffer, partialPath: string): Promise<string> { | ||
const fullPath = path.join(this.envSubDir, 'build', partialPath) | ||
return this.putFile(file, fullPath) | ||
} | ||
|
||
prependPath(partialPath: string) { | ||
return path.join(this.envSubDir, 'store', partialPath) | ||
} | ||
getPublicFileLocation(fullPath: string) { | ||
return encodeURI(`${this.baseUrl}${fullPath}`) | ||
} | ||
async checkExists(partialPath: string) { | ||
const fullPath = encodeURIComponent(this.prependPath(partialPath)) | ||
const url = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o/${fullPath}` | ||
const res = await fetch(url) | ||
return res.status !== 404 | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 10 additions & 13 deletions
23
packages/server/graphql/mutations/helpers/getTemplateIllustrationUrl.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,22 +1,19 @@ | ||
import getFileStoreManager from '../../../fileStorage/getFileStoreManager' | ||
|
||
// this can be removed after we sunset the old meeting template UI | ||
type CDN_TYPE = 'local' | 's3' | 'gcs' | ||
|
||
const getTemplateIllustrationUrl = (filename: string) => { | ||
const cdnType = process.env.FILE_STORE_PROVIDER | ||
const cdnType = process.env.FILE_STORE_PROVIDER as CDN_TYPE | undefined | ||
const manager = getFileStoreManager() | ||
|
||
const partialPath = `Organization/aGhostOrg/template/${filename}` | ||
if (cdnType === 'local') { | ||
return `/self-hosted/${partialPath}` | ||
} else if (cdnType === 's3') { | ||
const {AWS_S3_BUCKET, CDN_BASE_URL} = process.env | ||
if (!CDN_BASE_URL) throw new Error('Missing Env: CDN_BASE_URL') | ||
const baseUrl = new URL(CDN_BASE_URL.replace(/^\/+/, 'https://')) | ||
const {hostname, pathname} = baseUrl | ||
if (!hostname || !pathname) { | ||
throw new Error('CDN_BASE_URL ENV VAR IS INVALID') | ||
} | ||
const environment = pathname.replace(/^\//, '') | ||
if (!AWS_S3_BUCKET) throw new Error('Missing Env: AWS_S3_BUCKET') | ||
return `https://${AWS_S3_BUCKET}/${environment}/store/${partialPath}` | ||
} else { | ||
const fullPath = manager.prependPath(partialPath) | ||
return manager.getPublicFileLocation(fullPath) | ||
} | ||
throw new Error('Mssing Env: FILE_STORE_PROVIDER') | ||
} | ||
|
||
export default getTemplateIllustrationUrl |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
i don't like mutating past migrations, but this logic doesn't change anything, it just adds support for gcs & any future providers