From 6f6b4f31cd8db0a6c7d5782fcebeba3eafe6bda0 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 22 Mar 2024 12:21:01 +0000 Subject: [PATCH] databricks: lower file upload concurrency databricks driver loads files to memory in full, causing OOM issues. This is a temporary measure until we fix this --- materialize-databricks/staged_file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/materialize-databricks/staged_file.go b/materialize-databricks/staged_file.go index 01f1a28df3..31df8f66fa 100644 --- a/materialize-databricks/staged_file.go +++ b/materialize-databricks/staged_file.go @@ -16,7 +16,7 @@ import ( ) const fileSizeLimit = 128 * 1024 * 1024 -const uploadConcurrency = 5 // that means we may use up to 1.28GB disk space +const uploadConcurrency = 3 // fileBuffer provides Close() for a *bufio.Writer writing to an *os.File. Close() will flush the // buffer and close the underlying file.