Skip to content

Commit

Permalink
Implement asset:put endpoint for asset upload
Browse files Browse the repository at this point in the history 
Ref. SkygearIO#107
  • Loading branch information
Ben Lei committed Aug 31, 2016
1 parent 752d990 commit 2c08558
Show file tree
Hide file tree
Showing 5 changed files with 875 additions and 548 deletions.
10 changes: 6 additions & 4 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,8 @@ func main() {
r.Map("auth:logout", injector.Inject(&handler.LogoutHandler{}))
r.Map("auth:password", injector.Inject(&handler.PasswordHandler{}))

r.Map("asset:put", injector.Inject(&handler.AssetUploadHandler{}))

r.Map("record:fetch", injector.Inject(&handler.RecordFetchHandler{}))
r.Map("record:query", injector.Inject(&handler.RecordQueryHandler{}))
r.Map("record:save", injector.Inject(&handler.RecordSaveHandler{}))
Expand Down Expand Up @@ -211,11 +213,11 @@ func main() {
}))

fileGateway := router.NewGateway("files/(.+)", "/files/", serveMux)
fileGateway.GET(injector.Inject(&handler.AssetGetURLHandler{}))
fileGateway.GET(injector.Inject(&handler.GetFileHandler{}))

assetUploadURLHandler := injector.Inject(&handler.AssetUploadURLHandler{})
fileGateway.PUT(assetUploadURLHandler)
fileGateway.POST(assetUploadURLHandler)
uploadFileHandler := injector.Inject(&handler.UploadFileHandler{})
fileGateway.PUT(uploadFileHandler)
fileGateway.POST(uploadFileHandler)

corsHost := config.App.CORSHost

Expand Down
309 changes: 51 additions & 258 deletions pkg/server/handler/asset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,8 @@
package handler

import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"

skyAsset "github.com/skygeario/skygear-server/pkg/server/asset"
"github.com/skygeario/skygear-server/pkg/server/router"
Expand All @@ -35,303 +25,106 @@ import (
"github.com/skygeario/skygear-server/pkg/server/skyerr"
)

// used to clean file path
var sanitizedPathRe = regexp.MustCompile(`\A[/.]+`)

func clean(p string) string {
sanitized := strings.Replace(sanitizedPathRe.ReplaceAllString(path.Clean(p), ""), "..", "", -1)
// refs #426: S3 Asset Store is not able to put filename with `+` correctly
sanitized = strings.Replace(sanitized, "+", "", -1)
return sanitized
}

func validateAssetGetRequest(assetStore skyAsset.Store, fileName string, expiredAtUnix int64, signature string) skyerr.Error {
// check whether the request is expired
expiredAt := time.Unix(expiredAtUnix, 0)
if timeNow().After(expiredAt) {
return skyerr.NewError(skyerr.PermissionDenied, "Access denied")
}

// check the signature of the URL
signatureParser := assetStore.(skyAsset.SignatureParser)
valid, err := signatureParser.ParseSignature(signature, fileName, expiredAt)
if err != nil {
log.Errorf("Failed to parse signature: %v", err)

return skyerr.NewError(skyerr.PermissionDenied, "Access denied")
}

if !valid {
return skyerr.NewError(skyerr.InvalidSignature, "Invalid signature")
}
return nil
}

// AssetGetURLHandler models the get handler for asset
type AssetGetURLHandler struct {
AssetStore skyAsset.Store `inject:"AssetStore"`
DBConn router.Processor `preprocessor:"dbconn"`
preprocessors []router.Processor
}

// Setup sets preprocessors being used
func (h *AssetGetURLHandler) Setup() {
h.preprocessors = []router.Processor{
h.DBConn,
}
}

// GetPreprocessors returns all preprocessors
func (h *AssetGetURLHandler) GetPreprocessors() []router.Processor {
return h.preprocessors
}

// Handle handles the get request for asset
func (h *AssetGetURLHandler) Handle(payload *router.Payload, response *router.Response) {
payload.Req.ParseForm()

store := h.AssetStore
fileName := clean(payload.Params[0])
if store.(skyAsset.URLSigner).IsSignatureRequired() {
expiredAtUnix, err := strconv.ParseInt(payload.Req.Form.Get("expiredAt"), 10, 64)
if err != nil {
response.Err = skyerr.NewError(skyerr.InvalidArgument, "expect expiredAt to be an integer")
return
}

signature := payload.Req.Form.Get("signature")
requestErr := validateAssetGetRequest(h.AssetStore, fileName, expiredAtUnix, signature)
if requestErr != nil {
response.Err = requestErr
return
}
}

// everything's right, proceed with the request

conn := payload.DBConn
asset := skydb.Asset{}
if err := conn.GetAsset(fileName, &asset); err != nil {
log.Errorf("Failed to get asset: %v", err)

response.Err = skyerr.NewResourceFetchFailureErr("asset", fileName)
return
}

response.Header().Set("Content-Type", asset.ContentType)
response.Header().Set("Content-Length", strconv.FormatInt(asset.Size, 10))

reader, err := store.GetFileReader(fileName)
if err != nil {
log.Errorf("Failed to get file reader: %v", err)

response.Err = skyerr.NewResourceFetchFailureErr("asset", fileName)
return
}
defer reader.Close()

if _, err := io.Copy(response, reader); err != nil {
// there is nothing we can do if error occurred after started
// writing a response. Log.
log.Errorf("Error writing file to response: %v", err)
}
}

// AssetUploadURLHandler receives and persists a file to be associated by Record.
//
// Example curl (PUT):
// curl -XPUT \
// -H 'X-Skygear-API-Key: apiKey' \
// -H 'Content-Type: text/plain' \
// --data-binary '@file.txt' \
// http://localhost:3000/files/filename
//
// Example curl (POST):
// curl -XPOST \
// -H "X-Skygear-API-Key: apiKey" \
// -F 'file=@file.txt' \
// http://localhost:3000/files/filename
//
type AssetUploadURLHandler struct {
// AssetUploadHandler models the handler for asset upload request
type AssetUploadHandler struct {
AssetStore skyAsset.Store `inject:"AssetStore"`
AccessKey router.Processor `preprocessor:"accesskey"`
DBConn router.Processor `preprocessor:"dbconn"`
preprocessors []router.Processor
}

type assetUploadRequest struct {
filename string
contentType string
fileReader io.Reader
// AssetUploadResponse models the response of asset upload request
type AssetUploadResponse struct {
PostRequest *skyAsset.PostFileRequest `json:"post-request"`
Asset *map[string]interface{} `json:"asset"`
}

// Setup sets preprocessors being used
func (h *AssetUploadURLHandler) Setup() {
// Setup adds injected pre-processors to preprocessors array
func (h *AssetUploadHandler) Setup() {
h.preprocessors = []router.Processor{
h.AccessKey,
h.DBConn,
}
}

// GetPreprocessors returns all preprocessors
func (h *AssetUploadURLHandler) GetPreprocessors() []router.Processor {
// GetPreprocessors returns all pre-processors for the handler
func (h *AssetUploadHandler) GetPreprocessors() []router.Processor {
return h.preprocessors
}

// Handle handles the upload asset request
func (h *AssetUploadURLHandler) Handle(
// Handle is the handling method of the asset upload request
func (h *AssetUploadHandler) Handle(
payload *router.Payload,
response *router.Response,
) {

uploadRequest, err := parseUploadRequest(payload)
if err != nil {
response.Err = skyerr.NewError(skyerr.BadRequest, err.Error())
filename, ok := payload.Data["filename"].(string)
if !ok {
response.Err = skyerr.NewInvalidArgument(
"Missing filename or filename is invalid",
[]string{"filename"},
)
return
}

dir, file := filepath.Split(uploadRequest.filename)
file = fmt.Sprintf("%s-%s", uuidNew(), file)

fileName := filepath.Join(dir, file)
contentType := uploadRequest.contentType

if contentType == "" {
response.Err = skyerr.NewError(
skyerr.InvalidArgument,
"Content-Type cannot be empty",
contentType, ok := payload.Data["content-type"].(string)
if !ok {
response.Err = skyerr.NewInvalidArgument(
"Missing content type or content type is invalid",
[]string{"content-type"},
)
return
}

written, tempFile, err := copyToTempFile(uploadRequest.fileReader)
if err != nil {
response.Err = skyerr.MakeError(err)
contentSizeFloat, ok := payload.Data["content-size"].(float64)
if !ok {
response.Err = skyerr.NewInvalidArgument(
"Missing content size or content size is invalid",
[]string{"content-size"},
)
return
}
defer func() {
tempFile.Close()
os.Remove(tempFile.Name())
}()
contentSize := int64(contentSizeFloat)

if written == 0 {
response.Err = skyerr.NewError(skyerr.InvalidArgument, "Zero-byte content")
return
}
// Add UUID to Filename
dir, file := filepath.Split(filename)
file = strings.Join([]string{uuidNew(), file}, "-")
filename = filepath.Join(dir, file)

// Generate POST File Request
assetStore := h.AssetStore
if err := assetStore.PutFileReader(fileName, tempFile, written, contentType); err != nil {
response.Err = skyerr.MakeError(err)
postRequest, err := assetStore.GeneratePostFileRequest(filename)
if err != nil {
response.Err = skyerr.NewError(
skyerr.UnexpectedError,
"Fail to generate post file request",
)
return
}

// Save Asset to DB
conn := payload.DBConn
asset := skydb.Asset{
Name: fileName,
Name: filename,
ContentType: contentType,
Size: written,
Size: contentSize,
}

conn := payload.DBConn
if err := conn.SaveAsset(&asset); err != nil {
response.Err = skyerr.NewResourceSaveFailureErrWithStringID("asset", asset.Name)
return
}

if signer, ok := h.AssetStore.(skyAsset.URLSigner); ok {
// Add Signer to Asset for Serialization
if signer, ok := assetStore.(skyAsset.URLSigner); ok {
asset.Signer = signer
} else {
log.Warnf("Failed to acquire asset URLSigner, please check configuration")
response.Err = skyerr.NewError(skyerr.UnexpectedError, "Failed to sign the url")
return
}
response.Result = skyconv.ToMap((*skyconv.MapAsset)(&asset))
}

// parseUploadRequest tries to parse the payload from router to be compatible
// with both PUT requests and multiparts POST request
func parseUploadRequest(payload *router.Payload) (*assetUploadRequest, error) {
httpRequest := payload.Req
method := httpRequest.Method

var (
filename, contentType string
fileReader io.ReadCloser
)

if method == http.MethodPost {
// use 100 MB max memory to parse the multiparts Form
err := httpRequest.ParseMultipartForm(100 << 20)
if err != nil {
log.
WithField("error", err).
Error("Fail to parse multiparts form for asset upload")

return nil, err
}
assetMap := skyconv.ToMap((*skyconv.MapAsset)(&asset))

form := httpRequest.MultipartForm
fileHeader := form.File["file"]
if fileHeader == nil || len(fileHeader) == 0 {
log.Error("Missing file in multiparts form")

return nil, errors.New("Missing file in multiparts form")
}

firstFileHeader := fileHeader[0]

filename = firstFileHeader.Filename
contentType = firstFileHeader.Header["Content-Type"][0]
fileReader, err = firstFileHeader.Open()
if err != nil {
return nil, err
}
} else if method == http.MethodPut {
filename = clean(payload.Params[0])
contentType = httpRequest.Header.Get("Content-Type")
fileReader = httpRequest.Body
} else {
return nil, errors.New(
"Method " + method + " is not supported",
)
}

return &assetUploadRequest{
filename: filename,
contentType: contentType,
fileReader: fileReader,
}, nil
}

func copyToTempFile(src io.Reader) (written int64, tempFile *os.File, err error) {
tempFile, err = ioutil.TempFile("", "")
if err != nil {
return
response.Result = &AssetUploadResponse{
PostRequest: postRequest,
Asset: &assetMap,
}
written, err = io.Copy(tempFile, src)
if err != nil {
cleanupFile(tempFile)
tempFile = nil
return
}
if _, err = tempFile.Seek(0, 0); err != nil {
cleanupFile(tempFile)
tempFile = nil
return
}
return
}

func cleanupFile(f *os.File) error {
closeErr := f.Close()
if closeErr != nil {
log.Errorf("Failed to close tempFile %s: %v", f.Name(), closeErr)
return closeErr
}

if err := os.Remove(f.Name()); err != nil {
log.Errorf("Failed to remove file %s: %v", f.Name(), err)
return err
}

return nil
}
Loading

0 comments on commit 2c08558

Please sign in to comment.