Merge pull request #568 from tus/feat/new-tus-protocol

Support for IETF's resumable upload draft
tus · Jul 25, 2023 · caa0877 · caa0877
2 parents b4ffdf4 + 254d340
commit caa0877
Show file tree

Hide file tree

Showing 9 changed files with 625 additions and 44 deletions.
diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go
@@ -59,6 +59,7 @@ var Flags struct {
 	TLSCertFile             string
 	TLSKeyFile              string
 	TLSMode                 string
+	ExperimentalProtocol    bool
 
 	CPUProfile string
 }
@@ -108,6 +109,8 @@ func ParseFlags() {
 	flag.StringVar(&Flags.TLSCertFile, "tls-certificate", "", "Path to the file containing the x509 TLS certificate to be used. The file should also contain any intermediate certificates and the CA certificate.")
 	flag.StringVar(&Flags.TLSKeyFile, "tls-key", "", "Path to the file containing the key for the TLS certificate.")
 	flag.StringVar(&Flags.TLSMode, "tls-mode", "tls12", "Specify which TLS mode to use; valid modes are tls13, tls12, and tls12-strong.")
+	flag.BoolVar(&Flags.ExperimentalProtocol, "enable-experimental-protocol", false, "Enable support for the new resumable upload protocol draft from the IETF's HTTP working group, next to the current tus v1 protocol. (experimental and may be removed/changed in the future)")
+
 	flag.StringVar(&Flags.CPUProfile, "cpuprofile", "", "write cpu profile to file")
 	flag.Parse()
 

diff --git a/cmd/tusd/cli/serve.go b/cmd/tusd/cli/serve.go
@@ -24,17 +24,18 @@ const (
 // is put in place.
 func Serve() {
 	config := handler.Config{
-		MaxSize:                 Flags.MaxSize,
-		BasePath:                Flags.Basepath,
-		RespectForwardedHeaders: Flags.BehindProxy,
-		DisableDownload:         Flags.DisableDownload,
-		DisableTermination:      Flags.DisableTermination,
-		DisableCors:             Flags.DisableCors,
-		StoreComposer:           Composer,
-		NotifyCompleteUploads:   true,
-		NotifyTerminatedUploads: true,
-		NotifyUploadProgress:    true,
-		NotifyCreatedUploads:    true,
+		MaxSize:                    Flags.MaxSize,
+		BasePath:                   Flags.Basepath,
+		RespectForwardedHeaders:    Flags.BehindProxy,
+		EnableExperimentalProtocol: Flags.ExperimentalProtocol,
+		DisableDownload:            Flags.DisableDownload,
+		DisableTermination:         Flags.DisableTermination,
+		DisableCors:                Flags.DisableCors,
+		StoreComposer:              Composer,
+		NotifyCompleteUploads:      true,
+		NotifyTerminatedUploads:    true,
+		NotifyUploadProgress:       true,
+		NotifyCreatedUploads:       true,
 	}
 
 	if err := SetupPreHooks(&config); err != nil {

diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go
@@ -49,7 +49,7 @@ func (store FileStore) UseIn(composer *handler.StoreComposer) {
 }
 
 func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) {
-	if info.ID == "" { 
+	if info.ID == "" {
 		info.ID = uid.Uid()
 	}
 	binPath := store.binPath(info.ID)

diff --git a/pkg/handler/config.go b/pkg/handler/config.go
@@ -22,6 +22,10 @@ type Config struct {
 	// absolute URL containing a scheme, e.g. "http://tus.io"
 	BasePath string
 	isAbs    bool
+	// EnableExperimentalProtocol controls whether the new resumable upload protocol draft
+	// from the IETF's HTTP working group is accepted next to the current tus v1 protocol.
+	// See https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/
+	EnableExperimentalProtocol bool
 	// DisableDownload indicates whether the server will refuse downloads of the
 	// uploaded file, by not mounting the GET handler.
 	DisableDownload bool

diff --git a/pkg/handler/cors_test.go b/pkg/handler/cors_test.go
@@ -21,7 +21,7 @@ func TestCORS(t *testing.T) {
 			},
 			Code: http.StatusOK,
 			ResHeader: map[string]string{
-				"Access-Control-Allow-Headers": "Authorization, Origin, X-Requested-With, X-Request-ID, X-HTTP-Method-Override, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata, Upload-Defer-Length, Upload-Concat",
+				"Access-Control-Allow-Headers": "Authorization, Origin, X-Requested-With, X-Request-ID, X-HTTP-Method-Override, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata, Upload-Defer-Length, Upload-Concat, Upload-Incomplete",
 				"Access-Control-Allow-Methods": "POST, HEAD, PATCH, OPTIONS, GET, DELETE",
 				"Access-Control-Max-Age":       "86400",
 				"Access-Control-Allow-Origin":  "tus.io",
@@ -43,7 +43,7 @@ func TestCORS(t *testing.T) {
 			},
 			Code: http.StatusOK,
 			ResHeader: map[string]string{
-				"Access-Control-Allow-Headers": "Authorization, Origin, X-Requested-With, X-Request-ID, X-HTTP-Method-Override, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata, Upload-Defer-Length, Upload-Concat",
+				"Access-Control-Allow-Headers": "Authorization, Origin, X-Requested-With, X-Request-ID, X-HTTP-Method-Override, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata, Upload-Defer-Length, Upload-Concat, Upload-Incomplete",
 				"Access-Control-Allow-Methods": "POST, HEAD, PATCH, OPTIONS",
 				"Access-Control-Max-Age":       "86400",
 				"Access-Control-Allow-Origin":  "tus.io",
@@ -64,7 +64,7 @@ func TestCORS(t *testing.T) {
 			},
 			Code: http.StatusMethodNotAllowed,
 			ResHeader: map[string]string{
-				"Access-Control-Expose-Headers": "Upload-Offset, Location, Upload-Length, Tus-Version, Tus-Resumable, Tus-Max-Size, Tus-Extension, Upload-Metadata, Upload-Defer-Length, Upload-Concat",
+				"Access-Control-Expose-Headers": "Upload-Offset, Location, Upload-Length, Tus-Version, Tus-Resumable, Tus-Max-Size, Tus-Extension, Upload-Metadata, Upload-Defer-Length, Upload-Concat, Upload-Incomplete",
 				"Access-Control-Allow-Origin":   "tus.io",
 			},
 		}).Run(handler, t)

diff --git a/pkg/handler/head_test.go b/pkg/handler/head_test.go
@@ -146,4 +146,107 @@ func TestHead(t *testing.T) {
 			},
 		}).Run(handler, t)
 	})
+
+	SubTest(t, "ExperimentalProtocol", func(t *testing.T, _ *MockFullDataStore, _ *StoreComposer) {
+		SubTest(t, "IncompleteUpload", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					SizeIsDeferred: false,
+					Size:           10,
+					Offset:         5,
+				}, nil),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "HEAD",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+				},
+				Code: http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Incomplete":            "?1",
+					"Upload-Offset":                "5",
+				},
+			}).Run(handler, t)
+		})
+
+		SubTest(t, "CompleteUpload", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					SizeIsDeferred: false,
+					Size:           10,
+					Offset:         10,
+				}, nil),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "HEAD",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+				},
+				Code: http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Incomplete":            "?0",
+					"Upload-Offset":                "10",
+				},
+			}).Run(handler, t)
+		})
+
+		SubTest(t, "DeferredLength", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					SizeIsDeferred: true,
+					Offset:         5,
+				}, nil),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "HEAD",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+				},
+				Code: http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Incomplete":            "?1",
+					"Upload-Offset":                "5",
+				},
+			}).Run(handler, t)
+		})
+	})
 }
diff --git a/pkg/handler/patch_test.go b/pkg/handler/patch_test.go
@@ -683,4 +683,161 @@ func TestPatch(t *testing.T) {
 			ResBody: "an error while reading the body\n",
 		}).Run(handler, t)
 	})
+
+	SubTest(t, "ExperimentalProtocol", func(t *testing.T, _ *MockFullDataStore, _ *StoreComposer) {
+		SubTest(t, "CompleteUploadWithKnownSize", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					ID:             "yes",
+					Offset:         5,
+					Size:           10,
+					SizeIsDeferred: false,
+				}, nil),
+				upload.EXPECT().WriteChunk(context.Background(), int64(5), NewReaderMatcher("hello")).Return(int64(5), nil),
+				upload.EXPECT().FinishUpload(context.Background()),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "PATCH",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Offset":                "5",
+					"Upload-Incomplete":            "?0",
+				},
+				ReqBody: strings.NewReader("hello"),
+				Code:    http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Offset": "10",
+				},
+			}).Run(handler, t)
+		})
+		SubTest(t, "CompleteUploadWithUnknownSize", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					ID:             "yes",
+					Offset:         5,
+					Size:           0,
+					SizeIsDeferred: true,
+				}, nil),
+				upload.EXPECT().WriteChunk(context.Background(), int64(5), NewReaderMatcher("hello")).Return(int64(5), nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					ID:             "yes",
+					Offset:         10,
+					Size:           0,
+					SizeIsDeferred: true,
+				}, nil),
+				store.EXPECT().AsLengthDeclarableUpload(upload).Return(upload),
+				upload.EXPECT().DeclareLength(context.Background(), int64(10)),
+				upload.EXPECT().FinishUpload(context.Background()),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "PATCH",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Offset":                "5",
+					"Upload-Incomplete":            "?0",
+				},
+				ReqBody: strings.NewReader("hello"),
+				Code:    http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Offset": "10",
+				},
+			}).Run(handler, t)
+		})
+		SubTest(t, "ContinueUploadWithKnownSize", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					ID:             "yes",
+					Offset:         5,
+					Size:           10,
+					SizeIsDeferred: false,
+				}, nil),
+				upload.EXPECT().WriteChunk(context.Background(), int64(5), NewReaderMatcher("hel")).Return(int64(3), nil),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "PATCH",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Offset":                "5",
+					"Upload-Incomplete":            "?1",
+				},
+				ReqBody: strings.NewReader("hel"),
+				Code:    http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Offset": "8",
+				},
+			}).Run(handler, t)
+		})
+		SubTest(t, "ContinueUploadWithUnknownSize", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			upload := NewMockFullUpload(ctrl)
+
+			gomock.InOrder(
+				store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
+				upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
+					ID:             "yes",
+					Offset:         5,
+					Size:           0,
+					SizeIsDeferred: true,
+				}, nil),
+				upload.EXPECT().WriteChunk(context.Background(), int64(5), NewReaderMatcher("hel")).Return(int64(3), nil),
+			)
+
+			handler, _ := NewHandler(Config{
+				StoreComposer:              composer,
+				EnableExperimentalProtocol: true,
+			})
+
+			(&httpTest{
+				Method: "PATCH",
+				URL:    "yes",
+				ReqHeader: map[string]string{
+					"Upload-Draft-Interop-Version": "3",
+					"Upload-Offset":                "5",
+					"Upload-Incomplete":            "?1",
+				},
+				ReqBody: strings.NewReader("hel"),
+				Code:    http.StatusNoContent,
+				ResHeader: map[string]string{
+					"Upload-Offset": "8",
+				},
+			}).Run(handler, t)
+		})
+	})
 }