fix extra empty file bug (#63)

* fix extra empty file bug * tests
ConduitIO · Jun 17, 2022 · f985ad3 · f985ad3
1 parent 380dcd2
commit f985ad3
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 0 deletions.
diff --git a/destination/destination.go b/destination/destination.go
@@ -110,6 +110,10 @@ func (d *Destination) Teardown(ctx context.Context) error {
 }
 
 func (d *Destination) Flush(ctx context.Context) error {
+	if len(d.Buffer) == 0 {
+		return nil
+	}
+
 	bufferedRecords := d.Buffer
 	d.Buffer = d.Buffer[:0]
 

diff --git a/destination/destination_test.go b/destination/destination_test.go
@@ -62,6 +62,16 @@ func TestLocalParquet(t *testing.T) {
 		}
 	}
 
+	err = destination.Flush(ctx)
+	if err != nil {
+		t.Fatalf("failed to Flush: %v", err)
+	}
+
+	err = destination.Teardown(ctx)
+	if err != nil {
+		t.Fatalf("failed to Teardown: %v", err)
+	}
+
 	// The code above should produce two files in the fixtures directory:
 	// - local-0001.parquet
 	// - local-0002.parquet
@@ -115,6 +125,16 @@ func TestLocalJSON(t *testing.T) {
 		}
 	}
 
+	err = destination.Flush(ctx)
+	if err != nil {
+		t.Fatalf("failed to Flush: %v", err)
+	}
+
+	err = destination.Teardown(ctx)
+	if err != nil {
+		t.Fatalf("failed to Teardown: %v", err)
+	}
+
 	// The code above should produce two files in the fixtures directory:
 	// - local-0001.json
 	// - local-0002.json
@@ -198,6 +218,17 @@ func TestS3Parquet(t *testing.T) {
 		t.Fatalf("Destination writer expected to be writer.S3, but is actually %+v", writer)
 	}
 
+	err = destination.Flush(ctx)
+	if err != nil {
+		t.Fatalf("failed to Flush: %v", err)
+	}
+
+	err = destination.Teardown(ctx)
+	if err != nil {
+		t.Fatalf("failed to Teardown: %v", err)
+	}
+
+	// check if only two files are written
 	if len(writer.FilesWritten) != 2 {
 		t.Fatalf("Expected writer to have written 2 files, got %d", len(writer.FilesWritten))
 	}