Skip to content

Commit 2854986

Browse files
srikanthccvsrikanthccv
authored
Stop wrting to v1 tag attributes table (#487)
See SigNoz/signoz#6616 Fixes #467
1 parent 3856ccb commit 2854986

File tree

2 files changed

+11
-124
lines changed

2 files changed

+11
-124
lines changed

‎exporter/clickhouselogsexporter/exporter.go

Lines changed: 8 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,6 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
296296
var insertLogsStmtV2 driver.Batch
297297
var insertResourcesStmtV2 driver.Batch
298298
var statement driver.Batch
299-
var tagStatement driver.Batch
300299
var tagStatementV2 driver.Batch
301300
var attributeKeysStmt driver.Batch
302301
var resourceKeysStmt driver.Batch
@@ -311,9 +310,6 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
311310
if statement != nil {
312311
_ = statement.Abort()
313312
}
314-
if tagStatement != nil {
315-
_ = tagStatement.Abort()
316-
}
317313
if insertLogsStmtV2 != nil {
318314
_ = insertLogsStmtV2.Abort()
319315
}
@@ -342,11 +338,6 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
342338
}
343339
}
344340

345-
tagStatement, err = e.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", databaseName, DISTRIBUTED_TAG_ATTRIBUTES), driver.WithReleaseConnection())
346-
if err != nil {
347-
return fmt.Errorf("PrepareTagBatch:%w", err)
348-
}
349-
350341
tagStatementV2, err = e.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", databaseName, DISTRIBUTED_TAG_ATTRIBUTES_V2), driver.WithReleaseConnection())
351342
if err != nil {
352343
return fmt.Errorf("PrepareTagBatchV2:%w", err)
@@ -385,7 +376,7 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
385376
}
386377
resourceJson := string(serializedRes)
387378

388-
err = e.addAttrsToTagStatement(tagStatement, tagStatementV2, attributeKeysStmt, resourceKeysStmt, utils.TagTypeResource, resources, e.useNewSchema, shouldSkipKeys)
379+
err = e.addAttrsToTagStatement(tagStatementV2, attributeKeysStmt, resourceKeysStmt, utils.TagTypeResource, resources, e.useNewSchema, shouldSkipKeys)
389380
if err != nil {
390381
return err
391382
}
@@ -401,7 +392,7 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
401392
scopeAttributes := attributesToSlice(scope.Attributes(), true)
402393
scopeMap := attributesToMap(scope.Attributes(), true)
403394

404-
err := e.addAttrsToTagStatement(tagStatement, tagStatementV2, attributeKeysStmt, resourceKeysStmt, utils.TagTypeScope, scopeAttributes, e.useNewSchema, shouldSkipKeys)
395+
err := e.addAttrsToTagStatement(tagStatementV2, attributeKeysStmt, resourceKeysStmt, utils.TagTypeScope, scopeAttributes, e.useNewSchema, shouldSkipKeys)
405396
if err != nil {
406397
return err
407398
}
@@ -445,7 +436,7 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
445436
attributes := attributesToSlice(r.Attributes(), false)
446437
attrsMap := attributesToMap(r.Attributes(), false)
447438

448-
err = e.addAttrsToTagStatement(tagStatement, tagStatementV2, attributeKeysStmt, resourceKeysStmt, utils.TagTypeAttribute, attributes, e.useNewSchema, shouldSkipKeys)
439+
err = e.addAttrsToTagStatement(tagStatementV2, attributeKeysStmt, resourceKeysStmt, utils.TagTypeAttribute, attributes, e.useNewSchema, shouldSkipKeys)
449440
if err != nil {
450441
return err
451442
}
@@ -581,20 +572,6 @@ func (e *clickhouseLogsExporter) pushToClickhouse(ctx context.Context, ld plog.L
581572
stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(usage.TagTenantKey, k), tag.Upsert(usage.TagExporterIdKey, e.id.String())}, ExporterSigNozSentLogRecords.M(int64(v.Count)), ExporterSigNozSentLogRecordsBytes.M(int64(v.Size)))
582573
}
583574

584-
// push tag attributes
585-
tagWriteStart := time.Now()
586-
err = tagStatement.Send()
587-
stats.RecordWithTags(ctx,
588-
[]tag.Mutator{
589-
tag.Upsert(exporterKey, pipeline.SignalLogs.String()),
590-
tag.Upsert(tableKey, DISTRIBUTED_TAG_ATTRIBUTES),
591-
},
592-
writeLatencyMillis.M(int64(time.Since(tagWriteStart).Milliseconds())),
593-
)
594-
if err != nil {
595-
return err
596-
}
597-
598575
return err
599576
}
600577
}
@@ -667,7 +644,6 @@ func (e *clickhouseLogsExporter) addAttrsToAttributeKeysStatement(
667644
}
668645

669646
func (e *clickhouseLogsExporter) addAttrsToTagStatement(
670-
statement driver.Batch,
671647
tagStatementV2 driver.Batch,
672648
attributeKeysStmt driver.Batch,
673649
resourceKeysStmt driver.Batch,
@@ -683,20 +659,8 @@ func (e *clickhouseLogsExporter) addAttrsToTagStatement(
683659
e.logger.Debug("key has been skipped", zap.String("key", key))
684660
continue
685661
}
686-
err := statement.Append(
687-
time.Now(),
688-
v,
689-
tagType,
690-
"string",
691-
attrs.StringValues[i],
692-
nil,
693-
nil,
694-
)
695-
if err != nil {
696-
return fmt.Errorf("could not append string attribute to batch, err: %w", err)
697-
}
698662
e.addAttrsToAttributeKeysStatement(attributeKeysStmt, resourceKeysStmt, v, tagType, utils.TagDataTypeString)
699-
err = tagStatementV2.Append(
663+
err := tagStatementV2.Append(
700664
unixMilli,
701665
v,
702666
tagType,
@@ -709,31 +673,14 @@ func (e *clickhouseLogsExporter) addAttrsToTagStatement(
709673
}
710674
}
711675

712-
intTypeName := "int64"
713-
if useNewSchema {
714-
intTypeName = "float64"
715-
}
716676
for i, v := range attrs.IntKeys {
717677
key := utils.MakeKeyForAttributeKeys(v, tagType, utils.TagDataTypeNumber)
718678
if _, ok := shouldSkipKeys[key]; ok {
719679
e.logger.Debug("key has been skipped", zap.String("key", key))
720680
continue
721681
}
722-
723-
err := statement.Append(
724-
time.Now(),
725-
v,
726-
tagType,
727-
intTypeName,
728-
nil,
729-
nil,
730-
attrs.IntValues[i],
731-
)
732-
if err != nil {
733-
return fmt.Errorf("could not append number attribute to batch, err: %w", err)
734-
}
735682
e.addAttrsToAttributeKeysStatement(attributeKeysStmt, resourceKeysStmt, v, tagType, utils.TagDataTypeNumber)
736-
err = tagStatementV2.Append(
683+
err := tagStatementV2.Append(
737684
unixMilli,
738685
v,
739686
tagType,
@@ -751,20 +698,8 @@ func (e *clickhouseLogsExporter) addAttrsToTagStatement(
751698
e.logger.Debug("key has been skipped", zap.String("key", key))
752699
continue
753700
}
754-
err := statement.Append(
755-
time.Now(),
756-
v,
757-
tagType,
758-
"float64",
759-
nil,
760-
nil,
761-
attrs.FloatValues[i],
762-
)
763-
if err != nil {
764-
return fmt.Errorf("could not append number attribute to batch, err: %w", err)
765-
}
766701
e.addAttrsToAttributeKeysStatement(attributeKeysStmt, resourceKeysStmt, v, tagType, utils.TagDataTypeNumber)
767-
err = tagStatementV2.Append(
702+
err := tagStatementV2.Append(
768703
unixMilli,
769704
v,
770705
tagType,
@@ -782,20 +717,9 @@ func (e *clickhouseLogsExporter) addAttrsToTagStatement(
782717
e.logger.Debug("key has been skipped", zap.String("key", key))
783718
continue
784719
}
785-
err := statement.Append(
786-
time.Now(),
787-
v,
788-
tagType,
789-
"bool",
790-
nil,
791-
nil,
792-
nil,
793-
)
794-
if err != nil {
795-
return fmt.Errorf("could not append bool attribute to batch, err: %w", err)
796-
}
720+
797721
e.addAttrsToAttributeKeysStatement(attributeKeysStmt, resourceKeysStmt, v, tagType, utils.TagDataTypeBool)
798-
err = tagStatementV2.Append(
722+
err := tagStatementV2.Append(
799723
unixMilli,
800724
v,
801725
tagType,

‎exporter/clickhousetracesexporter/writerV3.go

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package clickhousetracesexporter
22

33
import (
44
"context"
5-
"errors"
65
"fmt"
76
"strconv"
87
"sync"
@@ -137,7 +136,6 @@ func (w *SpanWriter) writeErrorBatchV3(ctx context.Context, batchSpans []*SpanV3
137136

138137
func (w *SpanWriter) writeTagBatchV3(ctx context.Context, batchSpans []*SpanV3) error {
139138
var tagKeyStatement driver.Batch
140-
var tagStatement driver.Batch
141139
var tagStatementV2 driver.Batch
142140
var err error
143141
var shouldSkipKeys map[string]shouldSkipKey
@@ -150,17 +148,10 @@ func (w *SpanWriter) writeTagBatchV3(ctx context.Context, batchSpans []*SpanV3)
150148
if tagKeyStatement != nil {
151149
_ = tagKeyStatement.Abort()
152150
}
153-
if tagStatement != nil {
154-
_ = tagStatement.Abort()
155-
}
156151
if tagStatementV2 != nil {
157152
_ = tagStatementV2.Abort()
158153
}
159154
}()
160-
tagStatement, err = w.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", w.traceDatabase, w.attributeTable), driver.WithReleaseConnection())
161-
if err != nil {
162-
return fmt.Errorf("could not prepare batch for span attributes table due to error: %w", err)
163-
}
164155
tagKeyStatement, err = w.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", w.traceDatabase, w.attributeKeyTable), driver.WithReleaseConnection())
165156
if err != nil {
166157
return fmt.Errorf("could not prepare batch for span attributes key table due to error: %w", err)
@@ -216,15 +207,6 @@ func (w *SpanWriter) writeTagBatchV3(ctx context.Context, batchSpans []*SpanV3)
216207
unixMilli := (int64(span.StartTimeUnixNano/1e6) / 3600000) * 3600000
217208

218209
if spanAttribute.DataType == "string" {
219-
err = tagStatement.Append(
220-
time.Unix(0, int64(span.StartTimeUnixNano)),
221-
spanAttribute.Key,
222-
spanAttribute.TagType,
223-
spanAttribute.DataType,
224-
spanAttribute.StringValue,
225-
nil,
226-
spanAttribute.IsColumn,
227-
)
228210

229211
if _, ok := shouldSkipKeys[v2Key]; !ok {
230212
err = tagStatementV2.Append(
@@ -238,15 +220,6 @@ func (w *SpanWriter) writeTagBatchV3(ctx context.Context, batchSpans []*SpanV3)
238220
}
239221

240222
} else if spanAttribute.DataType == "float64" {
241-
err = tagStatement.Append(
242-
time.Unix(0, int64(span.StartTimeUnixNano)),
243-
spanAttribute.Key,
244-
spanAttribute.TagType,
245-
spanAttribute.DataType,
246-
nil,
247-
spanAttribute.NumberValue,
248-
spanAttribute.IsColumn,
249-
)
250223
if _, ok = shouldSkipKeys[v2Key]; !ok {
251224
err = tagStatementV2.Append(
252225
unixMilli,
@@ -258,15 +231,6 @@ func (w *SpanWriter) writeTagBatchV3(ctx context.Context, batchSpans []*SpanV3)
258231
)
259232
}
260233
} else if spanAttribute.DataType == "bool" {
261-
err = tagStatement.Append(
262-
time.Unix(0, int64(span.StartTimeUnixNano)),
263-
spanAttribute.Key,
264-
spanAttribute.TagType,
265-
spanAttribute.DataType,
266-
nil,
267-
nil,
268-
spanAttribute.IsColumn,
269-
)
270234
if _, ok = shouldSkipKeys[v2Key]; !ok {
271235
err = tagStatementV2.Append(
272236
unixMilli,
@@ -285,17 +249,16 @@ func (w *SpanWriter) writeTagBatchV3(ctx context.Context, batchSpans []*SpanV3)
285249
}
286250

287251
tagStart := time.Now()
288-
err1 := tagStatement.Send()
289-
err2 := tagStatementV2.Send()
252+
err = tagStatementV2.Send()
290253
stats.RecordWithTags(ctx,
291254
[]tag.Mutator{
292255
tag.Upsert(exporterKey, pipeline.SignalTraces.String()),
293256
tag.Upsert(tableKey, w.attributeTable),
294257
},
295258
writeLatencyMillis.M(int64(time.Since(tagStart).Milliseconds())),
296259
)
297-
if err1 != nil || err2 != nil {
298-
return fmt.Errorf("could not write to span attributes table due to error: %w", errors.Join(err1, err2))
260+
if err != nil {
261+
return fmt.Errorf("could not write to span attributes table due to error: %w", err)
299262
}
300263

301264
tagKeyStart := time.Now()

0 commit comments

Comments
 (0)