reduce the buffer size of message queue

Signed-off-by: SimFG <[email protected]>
zilliztech · Sep 27, 2024 · 10ddbbe · 10ddbbe
1 parent 9c8f9c7
commit 10ddbbe
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 9 deletions.
diff --git a/core/reader/factory_api.go b/core/reader/factory_api.go
@@ -33,6 +33,8 @@ import (
 	"github.com/zilliztech/milvus-cdc/core/util"
 )
 
+const BufferSize = "4"
+
 type FactoryCreator interface {
 	NewPmsFactory(cfg *config.PulsarConfig) msgstream.Factory
 	NewKmsFactory(cfg *config.KafkaConfig) msgstream.Factory
@@ -74,8 +76,8 @@ func (d *DefaultFactoryCreator) NewPmsFactory(cfg *config.PulsarConfig) msgstrea
 				EnableClientMetrics: config.NewParamItem("false"),
 			},
 			MQCfg: paramtable.MQConfig{
-				ReceiveBufSize: config.NewParamItem("16"),
-				MQBufSize:      config.NewParamItem("16"),
+				ReceiveBufSize: config.NewParamItem(BufferSize),
+				MQBufSize:      config.NewParamItem(BufferSize),
 			},
 		},
 	)
@@ -96,8 +98,8 @@ func (d *DefaultFactoryCreator) NewKmsFactory(cfg *config.KafkaConfig) msgstream
 				KafkaUseSSL:         config.NewParamItem("false"),
 			},
 			MQCfg: paramtable.MQConfig{
-				ReceiveBufSize: config.NewParamItem("16"),
-				MQBufSize:      config.NewParamItem("16"),
+				ReceiveBufSize: config.NewParamItem(BufferSize),
+				MQBufSize:      config.NewParamItem(BufferSize),
 			},
 		},
 	)

diff --git a/deployment/docker/cdc-16.yaml b/deployment/docker/cdc-16.yaml
@@ -29,7 +29,7 @@ sourceConfig:
     tlsKeyPath: deployment/cert/client.key # path to your key file
     tlsCACertPath: deployment/cert/ca.pem # path to your CACert file
     tlsMinVersion: 1.3
-  readChanLen: 10
+  readChanLen: 4
   defaultPartitionName: _default
   replicateChan: by-dev-replicate-msg
   channelNum: 16

diff --git a/deployment/docker/cdc-4.yaml b/deployment/docker/cdc-4.yaml
@@ -29,7 +29,7 @@ sourceConfig:
     tlsKeyPath: deployment/cert/client.key # path to your key file
     tlsCACertPath: deployment/cert/ca.pem # path to your CACert file
     tlsMinVersion: 1.3
-  readChanLen: 10
+  readChanLen: 4
   defaultPartitionName: _default
   replicateChan: by-dev-replicate-msg
   channelNum: 4

diff --git a/deployment/docker/cdc.yaml b/deployment/docker/cdc.yaml
@@ -29,7 +29,7 @@ sourceConfig:
     tlsKeyPath: deployment/cert/client.key # path to your key file
     tlsCACertPath: deployment/cert/ca.pem # path to your CACert file
     tlsMinVersion: 1.3
-  readChanLen: 10
+  readChanLen: 4
   defaultPartitionName: _default
   replicateChan: by-dev-replicate-msg
   pulsar:

diff --git a/doc/cdc-usage.md b/doc/cdc-usage.md
@@ -79,7 +79,7 @@ sourceConfig:
   # default partition name
   defaultPartitionName: _default
   # read buffer length, mainly used for buffering if writing data to milvus-target is slow.
-  readChanLen: 10
+  readChanLen: 4
   # milvus replicate channel name, which is `{msgChannel.chanNamePrefix.cluster}/{msgChannel.chanNamePrefix.replicateMsg}` in the milvus.yaml file
   replicateChan: by-dev-replicate-msg
   # milvus-source mq config, which is pulsar or kafka

diff --git a/server/configs/cdc.yaml b/server/configs/cdc.yaml
@@ -29,7 +29,7 @@ sourceConfig:
     tlsKeyPath: deployment/cert/client.key # path to your key file
     tlsCACertPath: deployment/cert/ca.pem # path to your CACert file
     tlsMinVersion: 1.3
-  readChanLen: 10
+  readChanLen: 4
   defaultPartitionName: _default
   replicateChan: by-dev-replicate-msg
   pulsar: