increasing buffer to file size

projectdiscovery · May 10, 2024 · 1b5e022 · 1b5e022
1 parent 8eb548f
commit 1b5e022
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/file/file.go b/file/file.go
@@ -583,7 +583,14 @@ func DedupeLines(filename string) error {
 	seenLines := make(map[string]struct{})
 	var deduplicatedLines []string
 
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
 	scanner := bufio.NewScanner(file)
+	maxSize := int(info.Size())
+	buffer := make([]byte, 0, maxSize)
+	scanner.Buffer(buffer, maxSize)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if _, exists := seenLines[line]; !exists {