From 6ef7d90d32aa835992123307a22c3f3bd5ad6776 Mon Sep 17 00:00:00 2001 From: Abhishek Raj Date: Fri, 27 Oct 2023 20:39:08 +0530 Subject: [PATCH] feat: rewrite presign url generation (#75) * feat: re-wrote presign url generation * chore: bump go version --- .env-template | 4 +- .github/workflows/go.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- configs/cloud.go | 2 +- configs/configs.go | 3 + constants/constants.go | 17 +++- controllers/{processVideo.go => process.go} | 4 +- controllers/upload.go | 11 +-- go.mod | 2 +- routes/routes.go | 4 +- service/consumer.go | 2 +- service/image.go | 7 ++ utils/upload.go | 101 ++++++++++++++++++-- 13 files changed, 135 insertions(+), 26 deletions(-) rename controllers/{processVideo.go => process.go} (91%) create mode 100644 service/image.go diff --git a/.env-template b/.env-template index 0d0fcca..923543a 100644 --- a/.env-template +++ b/.env-template @@ -1,6 +1,7 @@ PORT=8080 RABBITMQ_URI=rabbitmq_uri MAX_CONCURRENT_PROCESSING=1 +CDN_BASE_URL=http://localhost GOOGLE_APPLICATION_CREDENTIALS=path of the json file GCP_BUCKET_NAME= @@ -12,4 +13,5 @@ AZURE_ACCESS_KEY=azure_access_key AWS_ACCESS_KEY_ID=YOUR_AKID AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY -AWS_SESSION_TOKEN=TOKEN \ No newline at end of file +AWS_SESSION_TOKEN=TOKEN +AWS_S3_BUCKET_NAME=NAME \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0302cf2..d326e90 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: 1.21 - name: Build run: go build -v diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d4e3c93..935edc9 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.19 + go-version: 1.21 - uses: actions/checkout@v3 - name: Tidy run: go mod tidy diff --git a/configs/cloud.go b/configs/cloud.go index 1c686f9..1a4d274 100644 --- a/configs/cloud.go +++ b/configs/cloud.go @@ -55,7 +55,7 @@ func getAWSSession() *session.Session { Credentials: credentials.NewStaticCredentials( EnvVar[AWS_ACCESS_KEY_ID], EnvVar[AWS_SECRET_ACCESS_KEY], - EnvVar[AWS_SESSION_TOKEN], + "", ), CredentialsChainVerboseErrors: aws.Bool(true), }) diff --git a/configs/configs.go b/configs/configs.go index 3feb2b2..d1e0c4d 100644 --- a/configs/configs.go +++ b/configs/configs.go @@ -25,6 +25,7 @@ const ( GCP_CREDS_JSON_BASE64 GCP_SERVICE_USER_CREDS_JSON_BASE64 MAX_CONCURRENT_PROCESSING + CDN_BASE_URL ) var configVars = map[CONFIG_KEY]string{ @@ -43,6 +44,7 @@ var configVars = map[CONFIG_KEY]string{ AWS_SECRET_ACCESS_KEY: "AWS_SECRET_ACCESS_KEY", AWS_SESSION_TOKEN: "AWS_SESSION_TOKEN", MAX_CONCURRENT_PROCESSING: "MAX_CONCURRENT_PROCESSING", + CDN_BASE_URL: "CDN_BASE_URL", } var EnvVar = map[CONFIG_KEY]string{ @@ -61,6 +63,7 @@ var EnvVar = map[CONFIG_KEY]string{ AWS_SECRET_ACCESS_KEY: "", AWS_SESSION_TOKEN: "", MAX_CONCURRENT_PROCESSING: "1", + CDN_BASE_URL: "", } func LoadEnv() { diff --git a/constants/constants.go b/constants/constants.go index 401a48e..dd0980a 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -12,13 +12,26 @@ const WRITE_TIMEOUT = 5 * time.Second const IDLE_TIMEOUT = 30 * time.Second // folders +type FOLDER_TYPE int + +const ( + Dashes FOLDER_TYPE = iota + Images + Audios +) + +var CloudContainerNames = map[FOLDER_TYPE]string{ + Dashes: "dashes", + Images: "i", + Audios: "a", +} + const DOWNLOAD_FILE_PATH_PREFIX = "assets/downloads" const OUTPUT_FILE_PATH_PREFIX = "assets/output" -const CLOUD_CONTAINER_NAME = "dashes" const DOWNLOAD_FOLDER_PERM = 0666 const AWS_ENDPOINT = "http://localhost:4566" -const PRESIGNED_URL_EXPIRATION = 1 * time.Hour +const PRESIGNED_URL_EXPIRATION = 24 * time.Hour // event queue const RABBIT_MQ_CHANNEL = "VideoProcessing" diff --git a/controllers/processVideo.go b/controllers/process.go similarity index 91% rename from controllers/processVideo.go rename to controllers/process.go index 0c60479..051fa9b 100644 --- a/controllers/processVideo.go +++ b/controllers/process.go @@ -10,7 +10,9 @@ import ( "github.com/gin-gonic/gin" ) -func ProcessVideo(c *gin.Context) { +type Process struct{} + +func (*Process) Video(c *gin.Context) { var request types.Video if err := c.ShouldBindJSON(&request); err != nil { diff --git a/controllers/upload.go b/controllers/upload.go index d82f4f1..b764c5d 100644 --- a/controllers/upload.go +++ b/controllers/upload.go @@ -1,7 +1,6 @@ package controllers import ( - "log" "net/http" "path/filepath" "zestream-server/utils" @@ -11,18 +10,18 @@ import ( func GetPresignedURL(c *gin.Context) { fileName := c.Query("fileName") + fileType := c.Query("fileType") + basePath := c.Query("basePath") extension := filepath.Ext(fileName) - if fileName == "" || extension == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "Missing required query parameter 'fileName' or 'fileName' provided without any extension"}) + if fileName == "" || extension == "" || fileType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Missing required query parameter 'fileName' or 'fileType'"}) return } videoID := utils.VideoIDGen(extension) - log.Println(videoID, "id") - - url := utils.GetSignedURL(videoID) + url := utils.GetSignedURL(videoID, fileType, basePath) if url == "" { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error generating presigned URL", "details": ""}) return diff --git a/go.mod b/go.mod index ac068f3..7d2499e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module zestream-server -go 1.19 +go 1.21 require ( cloud.google.com/go/storage v1.28.1 diff --git a/routes/routes.go b/routes/routes.go index 133ac12..944c840 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -28,8 +28,10 @@ func Init() *gin.Engine { r.GET("health", controllers.Ping) + process := new(controllers.Process) + // /api/v1 - apiV1.POST("video/process", controllers.ProcessVideo) + apiV1.POST("video/process", process.Video) apiV1.GET("url/presigned", controllers.GetPresignedURL) return r diff --git a/service/consumer.go b/service/consumer.go index 33382ea..7785048 100644 --- a/service/consumer.go +++ b/service/consumer.go @@ -83,7 +83,7 @@ func processVideo(video *types.Video, guard <-chan int) { generateDash(videoFileName, video.Watermark) - uploader := utils.GetUploader(constants.CLOUD_CONTAINER_NAME, video.ID) + uploader := utils.GetUploader(constants.CloudContainerNames[constants.Dashes], video.ID) outputDir, err := utils.GetOutputFilePathName(videoFileName, "") utils.LogErr(err) diff --git a/service/image.go b/service/image.go new file mode 100644 index 0000000..d3549c1 --- /dev/null +++ b/service/image.go @@ -0,0 +1,7 @@ +package service + +type Image struct{} + +func (*Image) Get() { + +} diff --git a/utils/upload.go b/utils/upload.go index ed554a4..e924781 100644 --- a/utils/upload.go +++ b/utils/upload.go @@ -33,8 +33,9 @@ type Uploader interface { func GetUploader(containerName string, videoId string) Uploader { cloudSession := configs.GetCloudSession() + isAWS, isGCP, isAzure := GetWhichCloudIsEnabled() - if configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "" { + if isAWS { return AwsUploader{ ContainerName: containerName, VideoId: videoId, @@ -42,7 +43,7 @@ func GetUploader(containerName string, videoId string) Uploader { } } - if configs.EnvVar[configs.GCP_PROJECT_ID] != "" { + if isGCP { return &GcpUploader{ ContainerName: containerName, VideoId: videoId, @@ -50,7 +51,7 @@ func GetUploader(containerName string, videoId string) Uploader { } } - if configs.EnvVar[configs.AZURE_ACCESS_KEY] != "" { + if isAzure { return AzureUploader{ ContainerName: containerName, VideoId: videoId, @@ -228,25 +229,50 @@ func (a AzureUploader) Upload(walker fileWalk) { } -func GetSignedURL(videoId string) string { +func GetWhichCloudIsEnabled() (bool, bool, bool) { if configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "" { - return generateAWSSignedURL(videoId) + return true, false, false } if configs.EnvVar[configs.GCP_BUCKET_NAME] != "" { - return generateGCPSignedURL(videoId) + return false, true, false + } + + if configs.EnvVar[configs.AZURE_ACCOUNT_NAME] != "" { + return false, false, true + } + + return false, false, false +} + +func GetSignedURL(videoId string, fileType string, basePath string) string { + isAWS, isGCP, isAzure := GetWhichCloudIsEnabled() + + filePath, err := GetCloudStoragePath(basePath, videoId, fileType) + LogErr(err) + + if isAWS { + return generateAWSSignedURL(filePath) + } + + if isGCP { + return generateGCPSignedURL(filePath) + } + + if isAzure { + return generateAzureSignedURL(filePath) } return "" } -func generateGCPSignedURL(videoId string) string { +func generateGCPSignedURL(filePath string) string { client := configs.GetGCPClient(true) bucket := client.Bucket(configs.EnvVar[configs.GCP_BUCKET_NAME]) expirationTime := time.Now().Add(constants.PRESIGNED_URL_EXPIRATION) - url, err := bucket.SignedURL(videoId, &storage.SignedURLOptions{ + url, err := bucket.SignedURL(filePath, &storage.SignedURLOptions{ Method: "GET", Expires: expirationTime, }) @@ -257,15 +283,16 @@ func generateGCPSignedURL(videoId string) string { return url } -func generateAWSSignedURL(videoId string) string { +func generateAWSSignedURL(filePath string) string { session := configs.GetCloudSession() client := s3.New(session.AWS) req, err := client.PutObjectRequest(&s3.PutObjectInput{ Bucket: aws.String(configs.EnvVar[configs.AWS_S3_BUCKET_NAME]), - Key: aws.String(videoId), + Key: aws.String(filePath), }) + if err != nil { log.Println(err) } @@ -277,3 +304,57 @@ func generateAWSSignedURL(videoId string) string { return url } + +func generateBlobSAS(blobURL azblob.BlobURL, permissions string) (string, error) { + session := configs.GetCloudSession() + client := session.Azure + + sasQueryParams, err := azblob.BlobSASSignatureValues{ + Protocol: azblob.SASProtocolHTTPS, + ExpiryTime: time.Now().Add(constants.PRESIGNED_URL_EXPIRATION), + Permissions: permissions, + Version: time.Now().Format(time.DateOnly), + }.NewSASQueryParameters(client) + + if err != nil { + return "", err + } + + sasURL := blobURL.URL() + sasURL.RawQuery = sasQueryParams.Encode() + log.Println(sasQueryParams.Encode()) + return sasURL.String(), nil +} + +func generateAzureSignedURL(filePath string) string { + accountName := configs.EnvVar[configs.AZURE_ACCOUNT_NAME] + accountKey := configs.EnvVar[configs.AZURE_ACCESS_KEY] + containerName := constants.CloudContainerNames[constants.Images] + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + fmt.Println("Failed to create shared key credential:", err) + return "" + } + + pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + blobURL := azblob.NewContainerURL(*URL, pipeline).NewBlobURL(filePath) + + permissions := "rw" + + sasURL, err := generateBlobSAS(blobURL, permissions) + if err != nil { + fmt.Println("Failed to generate pre-signed URL:", err) + return "" + } + + return sasURL +} + +func GetCloudStoragePath(basePath, fileName string, _ string) (string, error) { + url, err := url.JoinPath(basePath, fileName) + LogErr(err) + + return url, err +}