Skip to content

Commit 0544375

Browse files
marclopmarclop
committed
Set MaxPollPartitionBytes upper limit
Signed-off-by: Marc Lopez Rubio <[email protected]>
1 parent 422f99d commit 0544375

File tree

2 files changed

+66
-50
lines changed

2 files changed

+66
-50
lines changed

‎kafka/consumer.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,12 @@ func (cfg *ConsumerConfig) finalize() error {
154154
if cfg.BrokerMaxReadBytes < 0 {
155155
errs = append(errs, errors.New("kafka: broker max read bytes cannot be negative"))
156156
}
157+
if cfg.MaxPollPartitionBytes > 0 {
158+
if cfg.MaxPollPartitionBytes > 1<<30 {
159+
cfg.Logger.Info("kafka: MaxPollPartitionBytes exceeds 1GiB, setting to 1GiB")
160+
cfg.MaxPollPartitionBytes = 1 << 30
161+
}
162+
}
157163
if cfg.BrokerMaxReadBytes > 1<<30 {
158164
cfg.Logger.Info("kafka: BrokerMaxReadBytes exceeds 1GiB, setting to 1GiB")
159165
cfg.BrokerMaxReadBytes = 1 << 30

‎kafka/consumer_test.go‎

Lines changed: 60 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -771,11 +771,12 @@ func TestConsumerConfigFinalizer(t *testing.T) {
771771
}
772772
t.Run("MaxPollBytes set to 1 << 20", func(t *testing.T) {
773773
cfg := ConsumerConfig{
774-
CommonConfig: ccfg,
775-
Processor: proc,
776-
Topics: []apmqueue.Topic{"topic"},
777-
GroupID: "groupid",
778-
MaxPollBytes: 1 << 20,
774+
CommonConfig: ccfg,
775+
Processor: proc,
776+
Topics: []apmqueue.Topic{"topic"},
777+
GroupID: "groupid",
778+
MaxPollBytes: 1 << 20,
779+
MaxPollPartitionBytes: 1 << 20,
779780
}
780781
err := cfg.finalize()
781782
require.NoError(t, err)
@@ -785,20 +786,22 @@ func TestConsumerConfigFinalizer(t *testing.T) {
785786
cfg.Logger = nil
786787

787788
assert.Equal(t, ConsumerConfig{
788-
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
789-
Topics: []apmqueue.Topic{"topic"},
790-
GroupID: "groupid",
791-
MaxPollBytes: 1 << 20,
792-
BrokerMaxReadBytes: 1 << 21,
789+
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
790+
Topics: []apmqueue.Topic{"topic"},
791+
GroupID: "groupid",
792+
MaxPollBytes: 1 << 20,
793+
MaxPollPartitionBytes: 1 << 20,
794+
BrokerMaxReadBytes: 1 << 21,
793795
}, cfg)
794796
})
795797
t.Run("MaxPollBytes set to 1 << 28", func(t *testing.T) {
796798
cfg := ConsumerConfig{
797-
CommonConfig: ccfg,
798-
Processor: proc,
799-
Topics: []apmqueue.Topic{"topic"},
800-
GroupID: "groupid",
801-
MaxPollBytes: 1 << 28,
799+
CommonConfig: ccfg,
800+
Processor: proc,
801+
Topics: []apmqueue.Topic{"topic"},
802+
GroupID: "groupid",
803+
MaxPollBytes: 1 << 28,
804+
MaxPollPartitionBytes: 1 << 28,
802805
}
803806
err := cfg.finalize()
804807
require.NoError(t, err)
@@ -808,20 +811,22 @@ func TestConsumerConfigFinalizer(t *testing.T) {
808811
cfg.Logger = nil
809812

810813
assert.Equal(t, ConsumerConfig{
811-
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
812-
Topics: []apmqueue.Topic{"topic"},
813-
GroupID: "groupid",
814-
MaxPollBytes: 1 << 28,
815-
BrokerMaxReadBytes: 1 << 29,
814+
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
815+
Topics: []apmqueue.Topic{"topic"},
816+
GroupID: "groupid",
817+
MaxPollBytes: 1 << 28,
818+
MaxPollPartitionBytes: 1 << 28,
819+
BrokerMaxReadBytes: 1 << 29,
816820
}, cfg)
817821
})
818822
t.Run("MaxPollBytes set to 1 << 29", func(t *testing.T) {
819823
cfg := ConsumerConfig{
820-
CommonConfig: ccfg,
821-
Processor: proc,
822-
Topics: []apmqueue.Topic{"topic"},
823-
GroupID: "groupid",
824-
MaxPollBytes: 1 << 29,
824+
CommonConfig: ccfg,
825+
Processor: proc,
826+
Topics: []apmqueue.Topic{"topic"},
827+
GroupID: "groupid",
828+
MaxPollBytes: 1 << 29,
829+
MaxPollPartitionBytes: 1 << 29,
825830
}
826831
err := cfg.finalize()
827832
require.NoError(t, err)
@@ -831,20 +836,22 @@ func TestConsumerConfigFinalizer(t *testing.T) {
831836
cfg.Logger = nil
832837

833838
assert.Equal(t, ConsumerConfig{
834-
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
835-
Topics: []apmqueue.Topic{"topic"},
836-
GroupID: "groupid",
837-
MaxPollBytes: 1 << 29,
838-
BrokerMaxReadBytes: 1 << 30,
839+
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
840+
Topics: []apmqueue.Topic{"topic"},
841+
GroupID: "groupid",
842+
MaxPollBytes: 1 << 29,
843+
MaxPollPartitionBytes: 1 << 29,
844+
BrokerMaxReadBytes: 1 << 30,
839845
}, cfg)
840846
})
841847
t.Run("MaxPollBytes set to 1 << 30", func(t *testing.T) {
842848
cfg := ConsumerConfig{
843-
CommonConfig: ccfg,
844-
Processor: proc,
845-
Topics: []apmqueue.Topic{"topic"},
846-
GroupID: "groupid",
847-
MaxPollBytes: 1 << 30,
849+
CommonConfig: ccfg,
850+
Processor: proc,
851+
Topics: []apmqueue.Topic{"topic"},
852+
GroupID: "groupid",
853+
MaxPollBytes: 1 << 30,
854+
MaxPollPartitionBytes: 1 << 30,
848855
}
849856
err := cfg.finalize()
850857
require.NoError(t, err)
@@ -854,20 +861,22 @@ func TestConsumerConfigFinalizer(t *testing.T) {
854861
cfg.Logger = nil
855862

856863
assert.Equal(t, ConsumerConfig{
857-
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
858-
Topics: []apmqueue.Topic{"topic"},
859-
GroupID: "groupid",
860-
MaxPollBytes: 1 << 30,
861-
BrokerMaxReadBytes: 1 << 30,
864+
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
865+
Topics: []apmqueue.Topic{"topic"},
866+
GroupID: "groupid",
867+
MaxPollBytes: 1 << 30,
868+
MaxPollPartitionBytes: 1 << 30,
869+
BrokerMaxReadBytes: 1 << 30,
862870
}, cfg)
863871
})
864872
t.Run("MaxPollBytes set to 1 << 31-1", func(t *testing.T) {
865873
cfg := ConsumerConfig{
866-
CommonConfig: ccfg,
867-
Processor: proc,
868-
Topics: []apmqueue.Topic{"topic"},
869-
GroupID: "groupid",
870-
MaxPollBytes: 1<<31 - 1,
874+
CommonConfig: ccfg,
875+
Processor: proc,
876+
Topics: []apmqueue.Topic{"topic"},
877+
GroupID: "groupid",
878+
MaxPollBytes: 1<<31 - 1,
879+
MaxPollPartitionBytes: 1<<31 - 1,
871880
}
872881
err := cfg.finalize()
873882
require.NoError(t, err)
@@ -877,11 +886,12 @@ func TestConsumerConfigFinalizer(t *testing.T) {
877886
cfg.Logger = nil
878887

879888
assert.Equal(t, ConsumerConfig{
880-
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
881-
Topics: []apmqueue.Topic{"topic"},
882-
GroupID: "groupid",
883-
MaxPollBytes: 1 << 30,
884-
BrokerMaxReadBytes: 1 << 30,
889+
CommonConfig: CommonConfig{Brokers: []string{"localhost:9092"}},
890+
Topics: []apmqueue.Topic{"topic"},
891+
GroupID: "groupid",
892+
MaxPollBytes: 1 << 30,
893+
MaxPollPartitionBytes: 1 << 30,
894+
BrokerMaxReadBytes: 1 << 30,
885895
}, cfg)
886896
})
887897
}

0 commit comments

Comments
 (0)