From 6e608ad9c192c5925c17981f380c15213bc7886a Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 22 Mar 2024 12:24:10 +0000 Subject: [PATCH] staging put: stream file uploads instead of loading all to memory os.ReadFile reads all of the content of the file into a byte array in memory, which can cause memory consumption pressure for users. Instead, an os.File instance is itself a byte reader, and we can provide the file directly to http.NewRequest so it can read the file in chunks and upload it as a stream, thus not holding the whole file in memory. --- connection.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/connection.go b/connection.go index 041f05b..41e1f9c 100644 --- a/connection.go +++ b/connection.go @@ -1,7 +1,6 @@ package dbsql import ( - "bytes" "context" "database/sql/driver" "encoding/json" @@ -433,13 +432,13 @@ func (c *conn) handleStagingPut(ctx context.Context, presignedUrl string, header } client := &http.Client{} - dat, err := os.ReadFile(localFile) - + dat, err := os.Open(localFile) if err != nil { return dbsqlerrint.NewDriverError(ctx, "error reading local file", err) } + defer dat.Close() - req, _ := http.NewRequest("PUT", presignedUrl, bytes.NewReader(dat)) + req, _ := http.NewRequest("PUT", presignedUrl, dat) for k, v := range headers { req.Header.Set(k, v)