fix: larger scanner buffer (#3369)

Signed-off-by: Song Gao <[email protected]>
lf-edge · Nov 11, 2024 · bfbb96d · bfbb96d
1 parent d62d2ed
commit bfbb96d
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/internal/io/file/line_reader.go b/internal/io/file/line_reader.go
@@ -41,6 +41,8 @@ func (r *LineReader) Close() error {
 func CreateLineReader(ctx api.StreamContext, fileStream io.Reader, config *FileSourceConfig) (FormatReader, error) {
 	scanner := bufio.NewScanner(fileStream)
 	scanner.Split(bufio.ScanLines)
+	// 1MB buffer
+	scanner.Buffer(nil, 1<<20)
 
 	reader := &LineReader{}
 	reader.scanner = scanner