feat: rewrite presign url generation (#75)

* feat: re-wrote presign url generation

* chore: bump go version

.env-template

@@ -1,6 +1,7 @@
 PORT=8080
 RABBITMQ_URI=rabbitmq_uri
 MAX_CONCURRENT_PROCESSING=1
+CDN_BASE_URL=http://localhost
 
 GOOGLE_APPLICATION_CREDENTIALS=path of the json file
 GCP_BUCKET_NAME=
@@ -12,4 +13,5 @@ AZURE_ACCESS_KEY=azure_access_key
 
 AWS_ACCESS_KEY_ID=YOUR_AKID
 AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY
-AWS_SESSION_TOKEN=TOKEN
+AWS_SESSION_TOKEN=TOKEN
+AWS_S3_BUCKET_NAME=NAME

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19
+        go-version: 1.21
 
     - name: Build
       run: go build -v

.github/workflows/golangci-lint.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: 1.21
       - uses: actions/checkout@v3
       - name: Tidy
         run: go mod tidy

configs/cloud.go

@@ -55,7 +55,7 @@ func getAWSSession() *session.Session {
 		Credentials: credentials.NewStaticCredentials(
 			EnvVar[AWS_ACCESS_KEY_ID],
 			EnvVar[AWS_SECRET_ACCESS_KEY],
-			EnvVar[AWS_SESSION_TOKEN],
+			"",
 		),
 		CredentialsChainVerboseErrors: aws.Bool(true),
 	})

configs/configs.go

@@ -25,6 +25,7 @@ const (
 	GCP_CREDS_JSON_BASE64
 	GCP_SERVICE_USER_CREDS_JSON_BASE64
 	MAX_CONCURRENT_PROCESSING
+	CDN_BASE_URL
 )
 
 var configVars = map[CONFIG_KEY]string{
@@ -43,6 +44,7 @@ var configVars = map[CONFIG_KEY]string{
 	AWS_SECRET_ACCESS_KEY:              "AWS_SECRET_ACCESS_KEY",
 	AWS_SESSION_TOKEN:                  "AWS_SESSION_TOKEN",
 	MAX_CONCURRENT_PROCESSING:          "MAX_CONCURRENT_PROCESSING",
+	CDN_BASE_URL:                       "CDN_BASE_URL",
 }
 
 var EnvVar = map[CONFIG_KEY]string{
@@ -61,6 +63,7 @@ var EnvVar = map[CONFIG_KEY]string{
 	AWS_SECRET_ACCESS_KEY:              "",
 	AWS_SESSION_TOKEN:                  "",
 	MAX_CONCURRENT_PROCESSING:          "1",
+	CDN_BASE_URL:                       "",
 }
 
 func LoadEnv() {

constants/constants.go

@@ -12,13 +12,26 @@ const WRITE_TIMEOUT = 5 * time.Second
 const IDLE_TIMEOUT = 30 * time.Second
 
 // folders
+type FOLDER_TYPE int
+
+const (
+	Dashes FOLDER_TYPE = iota
+	Images
+	Audios
+)
+
+var CloudContainerNames = map[FOLDER_TYPE]string{
+	Dashes: "dashes",
+	Images: "i",
+	Audios: "a",
+}
+
 const DOWNLOAD_FILE_PATH_PREFIX = "assets/downloads"
 const OUTPUT_FILE_PATH_PREFIX = "assets/output"
-const CLOUD_CONTAINER_NAME = "dashes"
 const DOWNLOAD_FOLDER_PERM = 0666
 
 const AWS_ENDPOINT = "http://localhost:4566"
-const PRESIGNED_URL_EXPIRATION = 1 * time.Hour
+const PRESIGNED_URL_EXPIRATION = 24 * time.Hour
 
 // event queue
 const RABBIT_MQ_CHANNEL = "VideoProcessing"

controllers/processVideo.go → controllers/process.go

@@ -10,7 +10,9 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func ProcessVideo(c *gin.Context) {
+type Process struct{}
+
+func (*Process) Video(c *gin.Context) {
 	var request types.Video
 
 	if err := c.ShouldBindJSON(&request); err != nil {

controllers/upload.go

@@ -1,7 +1,6 @@
 package controllers
 
 import (
-	"log"
 	"net/http"
 	"path/filepath"
 	"zestream-server/utils"
@@ -11,18 +10,18 @@ import (
 
 func GetPresignedURL(c *gin.Context) {
 	fileName := c.Query("fileName")
+	fileType := c.Query("fileType")
+	basePath := c.Query("basePath")
 	extension := filepath.Ext(fileName)
 
-	if fileName == "" || extension == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "Missing required query parameter 'fileName' or 'fileName' provided without any extension"})
+	if fileName == "" || extension == "" || fileType == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Missing required query parameter 'fileName' or 'fileType'"})
 		return
 	}
 
 	videoID := utils.VideoIDGen(extension)
 
-	log.Println(videoID, "id")
-
-	url := utils.GetSignedURL(videoID)
+	url := utils.GetSignedURL(videoID, fileType, basePath)
 	if url == "" {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error generating presigned URL", "details": ""})
 		return

go.mod

@@ -1,6 +1,6 @@
 module zestream-server
 
-go 1.19
+go 1.21
 
 require (
 	cloud.google.com/go/storage v1.28.1

routes/routes.go

@@ -28,8 +28,10 @@ func Init() *gin.Engine {
 
 	r.GET("health", controllers.Ping)
 
+	process := new(controllers.Process)
+
 	// /api/v1
-	apiV1.POST("video/process", controllers.ProcessVideo)
+	apiV1.POST("video/process", process.Video)
 	apiV1.GET("url/presigned", controllers.GetPresignedURL)
 
 	return r

service/consumer.go

@@ -83,7 +83,7 @@ func processVideo(video *types.Video, guard <-chan int) {
 
 	generateDash(videoFileName, video.Watermark)
 
-	uploader := utils.GetUploader(constants.CLOUD_CONTAINER_NAME, video.ID)
+	uploader := utils.GetUploader(constants.CloudContainerNames[constants.Dashes], video.ID)
 
 	outputDir, err := utils.GetOutputFilePathName(videoFileName, "")
 	utils.LogErr(err)

service/image.go

@@ -0,0 +1,7 @@
+package service
+
+type Image struct{}
+
+func (*Image) Get() {
+
+}

utils/upload.go

@@ -33,24 +33,25 @@ type Uploader interface {
 func GetUploader(containerName string, videoId string) Uploader {
 
 	cloudSession := configs.GetCloudSession()
+	isAWS, isGCP, isAzure := GetWhichCloudIsEnabled()
 
-	if configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "" {
+	if isAWS {
 		return AwsUploader{
 			ContainerName: containerName,
 			VideoId:       videoId,
 			Session:       cloudSession.AWS,
 		}
 	}
 
-	if configs.EnvVar[configs.GCP_PROJECT_ID] != "" {
+	if isGCP {
 		return &GcpUploader{
 			ContainerName: containerName,
 			VideoId:       videoId,
 			Client:        cloudSession.GCP,
 		}
 	}
 
-	if configs.EnvVar[configs.AZURE_ACCESS_KEY] != "" {
+	if isAzure {
 		return AzureUploader{
 			ContainerName: containerName,
 			VideoId:       videoId,
@@ -228,25 +229,50 @@ func (a AzureUploader) Upload(walker fileWalk) {
 
 }
 
-func GetSignedURL(videoId string) string {
+func GetWhichCloudIsEnabled() (bool, bool, bool) {
 	if configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "" {
-		return generateAWSSignedURL(videoId)
+		return true, false, false
 	}
 
 	if configs.EnvVar[configs.GCP_BUCKET_NAME] != "" {
-		return generateGCPSignedURL(videoId)
+		return false, true, false
+	}
+
+	if configs.EnvVar[configs.AZURE_ACCOUNT_NAME] != "" {
+		return false, false, true
+	}
+
+	return false, false, false
+}
+
+func GetSignedURL(videoId string, fileType string, basePath string) string {
+	isAWS, isGCP, isAzure := GetWhichCloudIsEnabled()
+
+	filePath, err := GetCloudStoragePath(basePath, videoId, fileType)
+	LogErr(err)
+
+	if isAWS {
+		return generateAWSSignedURL(filePath)
+	}
+
+	if isGCP {
+		return generateGCPSignedURL(filePath)
+	}
+
+	if isAzure {
+		return generateAzureSignedURL(filePath)
 	}
 
 	return ""
 }
 
-func generateGCPSignedURL(videoId string) string {
+func generateGCPSignedURL(filePath string) string {
 	client := configs.GetGCPClient(true)
 	bucket := client.Bucket(configs.EnvVar[configs.GCP_BUCKET_NAME])
 
 	expirationTime := time.Now().Add(constants.PRESIGNED_URL_EXPIRATION)
 
-	url, err := bucket.SignedURL(videoId, &storage.SignedURLOptions{
+	url, err := bucket.SignedURL(filePath, &storage.SignedURLOptions{
 		Method:  "GET",
 		Expires: expirationTime,
 	})
@@ -257,15 +283,16 @@ func generateGCPSignedURL(videoId string) string {
 	return url
 }
 
-func generateAWSSignedURL(videoId string) string {
+func generateAWSSignedURL(filePath string) string {
 	session := configs.GetCloudSession()
 
 	client := s3.New(session.AWS)
 
 	req, err := client.PutObjectRequest(&s3.PutObjectInput{
 		Bucket: aws.String(configs.EnvVar[configs.AWS_S3_BUCKET_NAME]),
-		Key:    aws.String(videoId),
+		Key:    aws.String(filePath),
 	})
+
 	if err != nil {
 		log.Println(err)
 	}
@@ -277,3 +304,57 @@ func generateAWSSignedURL(videoId string) string {
 
 	return url
 }
+
+func generateBlobSAS(blobURL azblob.BlobURL, permissions string) (string, error) {
+	session := configs.GetCloudSession()
+	client := session.Azure
+
+	sasQueryParams, err := azblob.BlobSASSignatureValues{
+		Protocol:    azblob.SASProtocolHTTPS,
+		ExpiryTime:  time.Now().Add(constants.PRESIGNED_URL_EXPIRATION),
+		Permissions: permissions,
+		Version:     time.Now().Format(time.DateOnly),
+	}.NewSASQueryParameters(client)
+
+	if err != nil {
+		return "", err
+	}
+
+	sasURL := blobURL.URL()
+	sasURL.RawQuery = sasQueryParams.Encode()
+	log.Println(sasQueryParams.Encode())
+	return sasURL.String(), nil
+}
+
+func generateAzureSignedURL(filePath string) string {
+	accountName := configs.EnvVar[configs.AZURE_ACCOUNT_NAME]
+	accountKey := configs.EnvVar[configs.AZURE_ACCESS_KEY]
+	containerName := constants.CloudContainerNames[constants.Images]
+
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		fmt.Println("Failed to create shared key credential:", err)
+		return ""
+	}
+
+	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+	URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName))
+	blobURL := azblob.NewContainerURL(*URL, pipeline).NewBlobURL(filePath)
+
+	permissions := "rw"
+
+	sasURL, err := generateBlobSAS(blobURL, permissions)
+	if err != nil {
+		fmt.Println("Failed to generate pre-signed URL:", err)
+		return ""
+	}
+
+	return sasURL
+}
+
+func GetCloudStoragePath(basePath, fileName string, _ string) (string, error) {
+	url, err := url.JoinPath(basePath, fileName)
+	LogErr(err)
+
+	return url, err
+}