From bfbb96d43e65fda91d600a61a480753aedee7d6a Mon Sep 17 00:00:00 2001
From: Song Gao <disxiaofei@163.com>
Date: Mon, 11 Nov 2024 10:50:48 +0800
Subject: [PATCH] fix: larger scanner buffer (#3369)

Signed-off-by: Song Gao <disxiaofei@163.com>
---
 internal/io/file/line_reader.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/io/file/line_reader.go b/internal/io/file/line_reader.go
index 7e271d1a41..e6bd555d23 100644
--- a/internal/io/file/line_reader.go
+++ b/internal/io/file/line_reader.go
@@ -41,6 +41,8 @@ func (r *LineReader) Close() error {
 func CreateLineReader(ctx api.StreamContext, fileStream io.Reader, config *FileSourceConfig) (FormatReader, error) {
 	scanner := bufio.NewScanner(fileStream)
 	scanner.Split(bufio.ScanLines)
+	// 1MB buffer
+	scanner.Buffer(nil, 1<<20)
 
 	reader := &LineReader{}
 	reader.scanner = scanner