hooks: Allow pre-create to change upload ID, meta data and storage details (#962)

* hooks: Allow pre-create to change upload ID, meta data and storage details

* fixup! hooks: Allow pre-create to change upload ID, meta data and storage details

* hooks: Add support for ChangeFileInfo in gRPC

Author: Acconut

cmd/tusd/cli/hooks.go

@@ -18,10 +18,10 @@ func hookTypeInSlice(a hooks.HookType, list []hooks.HookType) bool {
 	return false
 }
 
-func preCreateCallback(event handler.HookEvent) (handler.HTTPResponse, error) {
+func preCreateCallback(event handler.HookEvent) (handler.HTTPResponse, handler.FileInfoChanges, error) {
 	ok, hookRes, err := invokeHookSync(hooks.HookPreCreate, event)
 	if !ok || err != nil {
-		return handler.HTTPResponse{}, err
+		return handler.HTTPResponse{}, handler.FileInfoChanges{}, err
 	}
 
 	httpRes := hookRes.HTTPResponse
@@ -32,10 +32,12 @@ func preCreateCallback(event handler.HookEvent) (handler.HTTPResponse, error) {
 		err := handler.ErrUploadRejectedByServer
 		err.HTTPResponse = err.HTTPResponse.MergeWith(httpRes)
 
-		return handler.HTTPResponse{}, err
+		return handler.HTTPResponse{}, handler.FileInfoChanges{}, err
 	}
 
-	return httpRes, nil
+	// Pass any changes regarding file info from the hook to the handler.
+	changes := hookRes.ChangeFileInfo
+	return httpRes, changes, nil
 }
 
 func preFinishCallback(event handler.HookEvent) (handler.HTTPResponse, error) {

cmd/tusd/cli/hooks/grpc.go

@@ -94,5 +94,12 @@ func unmarshal(res *pb.HookResponse) (hookRes HookResponse) {
 		hookRes.HTTPResponse.Body = httpRes.Body
 	}
 
+	changes := res.ChangeFileInfo
+	if changes != nil {
+		hookRes.ChangeFileInfo.ID = changes.Id
+		hookRes.ChangeFileInfo.MetaData = changes.MetaData
+		hookRes.ChangeFileInfo.Storage = changes.Storage
+	}
+
 	return hookRes
 }

cmd/tusd/cli/hooks/hooks.go

@@ -48,6 +48,13 @@ type HookResponse struct {
 	// to the client.
 	RejectUpload bool
 
+	// ChangeFileInfo can be set to change selected properties of an upload before
+	// it has been created. See the handler.FileInfoChanges type for more details.
+	// Changes are applied on a per-property basis, meaning that specifying just
+	// one property leaves all others unchanged.
+	// This value is only respected for pre-create hooks.
+	ChangeFileInfo handler.FileInfoChanges
+
 	// StopUpload will cause the upload to be stopped during a PATCH request.
 	// This value is only respected for post-receive hooks. For other hooks,
 	// it is ignored. Use the HTTPResponse field to send details about the stop

cmd/tusd/cli/hooks/proto/v2/hook.proto

@@ -1,13 +1,15 @@
 // If this file gets changed, you must recompile the generate package in pkg/proto.
 // To do this, install the Go protobuf toolchain as mentioned in
-// https://github.com/golang/protobuf#installation.
-// Then use following command to recompile it with gRPC support:
-//   protoc --go_out=plugins=grpc:../../../../../pkg/proto/ v2/hook.proto
+// https://grpc.io/docs/languages/go/quickstart/#prerequisites.
+// Then use following command from the repository's root to recompile it with gRPC support:
+//   protoc --go-grpc_out=./pkg/ --go_out=./pkg/ ./cmd/tusd/cli/hooks/proto/v2/hook.proto
 // In addition, it may be necessary to update the protobuf or gRPC dependencies as well.
 
 syntax = "proto3";
 package v2;
 
+option go_package = "proto/v2";
+
 // HookRequest contains the information about the hook type, the involved upload,
 // and causing HTTP request.
 message HookRequest {
@@ -56,6 +58,35 @@ message FileInfo {
 	map <string, string> storage = 9;
 }
 
+// FileInfoChanges collects changes the should be made to a FileInfo object. This
+// can be done using the PreUploadCreateCallback to modify certain properties before
+// an upload is created. Properties which should not be modified (e.g. Size or Offset)
+// are intentionally left out here.
+message FileInfoChanges {
+	// If ID is not empty, it will be passed to the data store, allowing
+	// hooks to influence the upload ID. Be aware that a data store is not required to
+	// respect a pre-defined upload ID and might overwrite or modify it. However,
+	// all data stores in the github.com/tus/tusd package do respect pre-defined IDs.
+	string id = 1;
+
+	// If MetaData is not nil, it replaces the entire user-defined meta data from
+	// the upload creation request. You can add custom meta data fields this way
+	// or ensure that only certain fields from the user-defined meta data are saved.
+	// If you want to retain only specific entries from the user-defined meta data, you must
+	// manually copy them into this MetaData field.
+	// If you do not want to store any meta data, set this field to an empty map (`MetaData{}`).
+	// If you want to keep the entire user-defined meta data, set this field to nil.
+	map <string, string> metaData = 2;
+
+	// If Storage is not nil, it is passed to the data store to allow for minor adjustments
+	// to the upload storage (e.g. destination file name). The details are specific for each
+	// data store and should be looked up in their respective documentation.
+	// Please be aware that this behavior is currently not supported by any data store in
+	// the github.com/tus/tusd package.
+	map <string, string> storage = 3;
+}
+
+
 // HTTPRequest contains basic details of an incoming HTTP request.
 message HTTPRequest {
 	// Method is the HTTP method, e.g. POST or PATCH.
@@ -87,6 +118,13 @@ message HookResponse {
 	// to the client.
 	bool rejectUpload = 2;
 
+	// ChangeFileInfo can be set to change selected properties of an upload before
+	// it has been created. See the handler.FileInfoChanges type for more details.
+	// Changes are applied on a per-property basis, meaning that specifying just
+	// one property leaves all others unchanged.
+	// This value is only respected for pre-create hooks.
+	FileInfoChanges changeFileInfo = 4;
+
 	// StopUpload will cause the upload to be stopped during a PATCH request.
 	// This value is only respected for post-receive hooks. For other hooks,
 	// it is ignored. Use the HTTPResponse field to send details about the stop
@@ -104,7 +142,6 @@ message HTTPResponse {
 	string body = 3;
 }
 
-
 // The hook service definition.
 service HookHandler {
   // InvokeHook is invoked for every hook that is executed. HookRequest contains the

examples/hooks/grpc/Makefile

@@ -1,2 +1,2 @@
-hook_pb2.py:
-	python -m grpc_tools.protoc --proto_path=../../../cmd/tusd/cli/hooks/proto/v2/ hook.proto --python_out=. --grpc_python_out=.
+hook_pb2.py: ../../../cmd/tusd/cli/hooks/proto/v2/hook.proto
+	python3 -m grpc_tools.protoc --proto_path=../../../cmd/tusd/cli/hooks/proto/v2/ hook.proto --python_out=. --grpc_python_out=.

examples/hooks/grpc/hook_pb2.py

@@ -27,7 +27,11 @@ class HookHandlerServicer(object):
     """
 
     def InvokeHook(self, request, context):
-        """Sends a hook
+        """InvokeHook is invoked for every hook that is executed. HookRequest contains the
+        corresponding information about the hook type, the involved upload, and
+        causing HTTP request.
+        The return value HookResponse allows to stop or reject an upload, as well as modifying
+        the HTTP response. See the documentation for HookResponse for more details.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')

examples/hooks/grpc/hook_pb2_grpc.py

@@ -1,6 +1,7 @@
 import grpc
 from concurrent import futures
 import time
+import uuid
 import hook_pb2_grpc as pb2_grpc
 import hook_pb2 as pb2
 
@@ -19,13 +20,22 @@ def InvokeHook(self, hook_request, context):
 
         # Example: Use the pre-create hook to check if a filename has been supplied
         # using metadata. If not, the upload is rejected with a custom HTTP response.
+        # In addition, a custom upload ID with a choosable prefix is supplied.
+        # Metadata is configured, so that it only retains the filename meta data
+        # and the creation time.
         if hook_request.type == 'pre-create':
-            filename = hook_request.event.upload.metaData['filename']
-            if filename == "":
+            metaData = hook_request.event.upload.metaData
+            isValid = 'filename' in metaData
+            if not isValid:
                 hook_response.rejectUpload = True
                 hook_response.httpResponse.statusCode = 400
                 hook_response.httpResponse.body = 'no filename provided'
                 hook_response.httpResponse.headers['X-Some-Header'] = 'yes'
+            else:
+                hook_response.changeFileInfo.id = f'prefix-{uuid.uuid4()}'
+                hook_response.changeFileInfo.metaData
+                hook_response.changeFileInfo.metaData['filename'] = metaData['filename']
+                hook_response.changeFileInfo.metaData['creation_time'] = time.ctime()
 
         # Example: Use the post-finish hook to print information about a completed upload,
         # including its storage location.