log/rollwriter: skip buffer and write directly when data packet exceeds expected size

Author: liuzengh

log/rollwriter/async_roll_writer.go

@@ -117,6 +117,17 @@ func (w *AsyncRollWriter) batchWriteLog() {
 				buffer.Reset()
 			}
 		case data := <-w.logQueue:
+			if len(data) >= w.opts.WriteLogSize {
+				// If the length of the current data exceeds the expected maximum value,
+				// we directly write it to the underlying logger instead of placing it into the buffer.
+				// This prevents the buffer from being overwhelmed by excessively large data,
+				// which could lead to memory leaks.
+				// Prior to that, we need to write the existing data in the buffer to the underlying logger.
+				_, _ = w.logger.Write(buffer.Bytes())
+				buffer.Reset()
+				_, _ = w.logger.Write(data)
+				continue
+			}
 			buffer.Write(data)
 			if buffer.Len() >= w.opts.WriteLogSize {
 				_, err := w.logger.Write(buffer.Bytes())

log/rollwriter/roll_writer_test.go

@@ -376,6 +376,16 @@ func TestAsyncRollWriterSyncTwice(t *testing.T) {
 	require.Nil(t, w.Close())
 }
 
+func TestAsyncRollWriterDirectWrite(t *testing.T) {
+	logSize := 1
+	w := NewAsyncRollWriter(&noopWriteCloser{}, WithWriteLogSize(logSize))
+	_, _ = w.Write([]byte("hello"))
+	time.Sleep(time.Millisecond)
+	require.Nil(t, w.Sync())
+	require.Nil(t, w.Sync())
+	require.Nil(t, w.Close())
+}
+
 func TestRollWriterError(t *testing.T) {
 	logDir := t.TempDir()
 	t.Run("reopen file", func(t *testing.T) {