Skip to content

Commit d035b1e

Browse files
sfc-gh-lkucharskisfc-gh-lkucharski
authored
FLOW-7097 Fix protobuf tests (#1237)
1 parent adf5352 commit d035b1e

File tree

6 files changed

+36
-36
lines changed

6 files changed

+36
-36
lines changed

‎test/rest_request_template/travis_correct_confluent_protobuf_protobuf.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
"key.converter":"io.confluent.connect.protobuf.ProtobufConverter",
1313
"key.converter.schema.registry.url":"CONFLUENT_SCHEMA_REGISTRY",
1414
"value.converter":"io.confluent.connect.protobuf.ProtobufConverter",
15-
"value.converter.schema.registry.url":"CONFLUENT_SCHEMA_REGISTRY"
15+
"value.converter.schema.registry.url":"CONFLUENT_SCHEMA_REGISTRY",
16+
"transforms": "add_record_content",
17+
"transforms.add_record_content.type": "org.apache.kafka.connect.transforms.HoistField$Value",
18+
"transforms.add_record_content.field": "RECORD_CONTENT"
1619
}
1720
}

‎test/rest_request_template/travis_correct_native_string_protobuf.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
"snowflake.role.name": "SNOWFLAKE_ROLE",
1212
"key.converter":"org.apache.kafka.connect.storage.StringConverter",
1313
"value.converter":"com.blueapron.connect.protobuf.ProtobufConverter",
14-
"value.converter.protoClassName":"com.snowflake.kafka.test.protobuf.SensorReadingImpl$SensorReading"
14+
"value.converter.protoClassName":"com.snowflake.kafka.test.protobuf.SensorReadingImpl$SensorReading",
15+
"transforms": "add_record_content",
16+
"transforms.add_record_content.type": "org.apache.kafka.connect.transforms.HoistField$Value",
17+
"transforms.add_record_content.field": "RECORD_CONTENT"
1518
}
1619
}

‎test/test_suit/test_avrosr_avrosr.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,5 @@ def verify(self, round):
9595
assert res['SOMEDOUBLENEGATIVEINFINITY'] == '-Inf'
9696
self.driver.regexMatchMeta(res['RECORD_METADATA'], goldMeta)
9797

98-
99-
def clean(self):
98+
def clean(self):
10099
self.driver.cleanTableStagePipe(self.topic)

‎test/test_suit/test_confluent_protobuf_protobuf.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ def __init__(self, driver, nameSalt):
4040
'value.serializer': self.valueProtobufSerializer}
4141

4242
self.protobufProducer = SerializingProducer(producer_conf)
43-
44-
self.driver.create_table(self.tableName)
43+
self.driver.snowflake_conn.cursor().execute(f"""create or replace table {self.tableName} (record_metadata variant, record_content variant)""")
4544

4645

4746
def getConfigFileName(self):
@@ -64,7 +63,6 @@ def verify(self, round):
6463
res = self.driver.snowflake_conn.cursor().execute(
6564
"Select * from {} limit 1".format(self.topic)).fetchone()
6665

67-
# "schema_id" is lost since they are using native avro converter
6866
goldMeta = r'{"CreateTime":\d*,"SnowflakeConnectorPushTime":\d*,"key":{"bytes_val":"3q0=","dateTime":1234,"device":' \
6967
r'{"deviceID":"555-4321","enabled":true},"double_array_val":' \
7068
r'[0.3333333333333333,32.21,4.343243210000000e+08],"float_val":4321.432,' \

‎test/test_suit/test_native_string_protobuf.py‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ def __init__(self, driver, nameSalt):
2424
self.sensor.bytes_val = b'\xDE\xAD'
2525
self.sensor.double_array_val.extend([1/3, 32.21, 434324321])
2626
self.sensor.uint64_val = (1 << 64) - 1
27-
28-
self.driver.create_table(self.tableName)
27+
28+
self.driver.snowflake_conn.cursor().execute(f"""create or replace table {self.tableName} (record_metadata variant, record_content variant)""")
2929

3030
def getConfigFileName(self):
3131
return self.fileName + ".json"
@@ -47,16 +47,13 @@ def verify(self, round):
4747
res = self.driver.snowflake_conn.cursor().execute(
4848
"Select * from {} limit 1".format(self.topic)).fetchone()
4949

50-
# "schema_id" is lost since they are using native avro converter
51-
goldMeta = r'{"CreateTime":\d+,"SnowflakeConnectorPushTime":\d+,""offset":0,"partition":0,' \
52-
r'"topic":"travis_correct_native_string_protobuf_\w+"}'
50+
goldMeta = r'{"CreateTime":\d*,"SnowflakeConnectorPushTime":\d*,"offset":0,"partition":0,' \
51+
r'"topic":"travis_correct_native_string_protobuf_\w*"}'
5352
goldContent = r'{"bytes_val":"3q0=","dateTime":1234,"device":{"deviceID":"555-4321","enabled":true},' \
5453
r'"double_array_val":[0.3333333333333333,32.21,4.343243210000000e+08],' \
5554
r'"float_val":4321.432,"int32_val":2147483647,"reading":321.321,"sint32_val":2147483647,' \
5655
r'"sint64_val":9223372036854775807,"uint32_val":4294967295,"uint64_val":18446744073709551615}'
5756
self.driver.regexMatchOneLine(res, goldMeta, goldContent)
5857

59-
60-
6158
def clean(self):
6259
self.driver.cleanTableStagePipe(self.topic)

‎test/test_suites.py‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121
)
2222
from test_suit.test_avro_avro import TestAvroAvro
2323
from test_suit.test_avrosr_avrosr import TestAvrosrAvrosr
24-
# https://snowflakecomputing.atlassian.net/browse/FLOW-7097
25-
#from test_suit.test_confluent_protobuf_protobuf import TestConfluentProtobufProtobuf
24+
25+
from test_suit.test_confluent_protobuf_protobuf import TestConfluentProtobufProtobuf
2626
from test_suit.test_json_json import TestJsonJson
2727
from test_suit.test_multiple_topic_to_one_table_snowpipe_streaming import (
2828
TestMultipleTopicToOneTableSnowpipeStreaming,
@@ -82,8 +82,8 @@
8282
from test_suit.test_snowpipe_streaming_string_json_ignore_tombstone import (
8383
TestSnowpipeStreamingStringJsonIgnoreTombstone,
8484
)
85-
# https://snowflakecomputing.atlassian.net/browse/FLOW-7097
86-
#from test_suit.test_native_string_protobuf import TestNativeStringProtobuf
85+
86+
from test_suit.test_native_string_protobuf import TestNativeStringProtobuf
8787
from test_suit.test_string_avro import TestStringAvro
8888
from test_suit.test_string_avrosr import TestStringAvrosr
8989
from test_suit.test_string_json import TestStringJson
@@ -219,24 +219,24 @@ def create_end_to_end_test_suites(driver, nameSalt, schemaRegistryAddress, testS
219219
# cloud_platform=CloudPlatform.ALL,
220220
# ),
221221
# ),
222-
# (
223-
# "TestNativeStringProtobuf",
224-
# EndToEndTestSuite(
225-
# test_instance=TestNativeStringProtobuf(driver, nameSalt),
226-
# run_in_confluent=True,
227-
# run_in_apache=True,
228-
# cloud_platform=CloudPlatform.ALL,
229-
# ),
230-
# ),
231-
# (
232-
# "TestConfluentProtobufProtobuf",
233-
# EndToEndTestSuite(
234-
# test_instance=TestConfluentProtobufProtobuf(driver, nameSalt),
235-
# run_in_confluent=False,
236-
# run_in_apache=False,
237-
# cloud_platform=CloudPlatform.ALL,
238-
# ),
239-
# ),
222+
(
223+
"TestNativeStringProtobuf",
224+
EndToEndTestSuite(
225+
test_instance=TestNativeStringProtobuf(driver, nameSalt),
226+
run_in_confluent=True,
227+
run_in_apache=True,
228+
cloud_platform=CloudPlatform.ALL,
229+
),
230+
),
231+
(
232+
"TestConfluentProtobufProtobuf",
233+
EndToEndTestSuite(
234+
test_instance=TestConfluentProtobufProtobuf(driver, nameSalt),
235+
run_in_confluent=False,
236+
run_in_apache=False,
237+
cloud_platform=CloudPlatform.ALL,
238+
),
239+
),
240240
(
241241
"TestNullableValuesAfterSmt",
242242
EndToEndTestSuite(

0 commit comments

Comments
 (0)