Skip to content

Commit 6a67ca6

Browse files
authored
Use golangci instead of go vet (#21)
* Use golangci instead of `go vet` * Address linters complaints
1 parent 6bbd6f9 commit 6a67ca6

19 files changed

+306
-2127
lines changed

.golangci.yml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
issues:
2+
exclude-rules:
3+
- path: examples/*
4+
linters:
5+
- forbidigo
6+
linters:
7+
disable-all: true
8+
enable:
9+
- errcheck
10+
- gocritic
11+
- gofumpt
12+
- goheader
13+
- goimports
14+
- gosimple
15+
- govet
16+
- ineffassign
17+
#- lll
18+
- misspell
19+
- predeclared
20+
- staticcheck
21+
- thelper
22+
- tparallel
23+
- typecheck
24+
- unused
25+
- forbidigo
26+
run:
27+
allow-parallel-runners: true
28+
modules-download-mode: readonly
29+
tests: true
30+
go: '1.14'
31+
linters-settings:
32+
govet:
33+
enable-all: true
34+
disable:
35+
- shadow
36+
- fieldalignment
37+
gofumpt:
38+
extra-rules: true
39+
goimports:
40+
local-prefixes: github.com/scylladb/scylla-cdc-go

Makefile

+43-4
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,31 @@ define dl_bin
4343
chmod +x "$(GOBIN)/$(1)";
4444
endef
4545

46+
define dl_tgz
47+
@if [ ! -f "$(GOBIN)/$(1)" ]; then \
48+
echo "Downloading $(GOBIN)/$(1)"; \
49+
curl --progress-bar -L $(2) | tar zxf - --wildcards --strip 1 -C $(GOBIN) '*/$(1)'; \
50+
chmod +x "$(GOBIN)/$(1)"; \
51+
fi
52+
endef
53+
54+
define dl_tgz
55+
@[ -d "$(GOBIN)" ] || mkdir -p "$(GOBIN)"; \
56+
if [ -L "$(GOBIN)/$(1)" ] && [ -e "$(GOBIN)/$(1)" ]; then \
57+
echo "$(GOBIN)/$(1) is already installed."; \
58+
return 0; \
59+
fi; \
60+
if $(GOBIN)/$(1) --version 2>/dev/null | grep "$(2)" >/dev/null; then \
61+
echo "$(GOBIN)/$(1) is already installed."; \
62+
return 0; \
63+
fi; \
64+
echo "$(GOBIN)/$(1) is not found, downloading."; \
65+
rm -f "$(GOBIN)/$(1)" >/dev/null 2>&1 \
66+
echo "Downloading $(GOBIN)/$(1)"; \
67+
curl --progress-bar -L $(3) | tar zxf - --wildcards --strip 1 -C $(GOBIN) '*/$(1)'; \
68+
chmod +x "$(GOBIN)/$(1)";
69+
endef
70+
4671
.PHONY: tune-aio-max-nr
4772
tune-aio-max-nr:
4873
@bash -c '[[ "2097152" -ge "$(cat /proc/sys/fs/aio-max-nr)" ]] && sudo sh -c "echo 2097152 >> /proc/sys/fs/aio-max-nr"'
@@ -74,6 +99,18 @@ else
7499
@exit 69
75100
endif
76101

102+
103+
install-golangci-lint: GOLANGCI_VERSION = 1.60.1
104+
install-golangci-lint: Makefile
105+
ifeq ($(GOARCH),arm64)
106+
$(call dl_tgz,golangci-lint,${GOLANGCI_VERSION},https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_VERSION)/golangci-lint-$(GOLANGCI_VERSION)-$(GOOS)-arm64.tar.gz)
107+
else ifeq ($(GOARCH),amd64)
108+
$(call dl_tgz,golangci-lint,${GOLANGCI_VERSION},https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_VERSION)/golangci-lint-$(GOLANGCI_VERSION)-$(GOOS)-amd64.tar.gz)
109+
else
110+
@printf 'Unknown architecture "%s"\n', "$(GOARCH)" \
111+
@exit 69
112+
endif
113+
77114
.PHONY: test
78115
test: install-docker-compose start-docker-environment
79116
@echo "Running tests"
@@ -85,10 +122,12 @@ build:
85122
@go build -v ./...
86123

87124
.PHONY: lint
88-
lint:
89-
@go vet -v ./...
125+
lint: install-golangci-lint
126+
go list ./... | sed -e 's/github.com\/scylladb\/scylla-cdc-go/./g' | \
127+
xargs $(GOBIN)/golangci-lint run --timeout=5m
90128

91129
.PHONY: fix-lint
92-
fix-lint:
93-
@go vet -v ./...
130+
fix-lint: install-golangci-lint
131+
go list ./... | sed -e 's/github.com\/scylladb\/scylla-cdc-go/./g' | \
132+
xargs $(GOBIN)/golangci-lint run --timeout=5m --fix
94133

change.go

+45-44
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,15 @@ type OperationType int8
1919

2020
const (
2121
PreImage OperationType = 0
22-
Update = 1
23-
Insert = 2
24-
RowDelete = 3
25-
PartitionDelete = 4
26-
RangeDeleteStartInclusive = 5
27-
RangeDeleteStartExclusive = 6
28-
RangeDeleteEndInclusive = 7
29-
RangeDeleteEndExclusive = 8
30-
PostImage = 9
22+
Update OperationType = 1
23+
Insert OperationType = 2
24+
RowDelete OperationType = 3
25+
PartitionDelete OperationType = 4
26+
RangeDeleteStartInclusive OperationType = 5
27+
RangeDeleteStartExclusive OperationType = 6
28+
RangeDeleteEndInclusive OperationType = 7
29+
RangeDeleteEndExclusive OperationType = 8
30+
PostImage OperationType = 9
3131
)
3232

3333
// String is needed to implement the fmt.Stringer interface.
@@ -512,6 +512,7 @@ type changeConsumerFuncInstance struct {
512512
func (ccfi *changeConsumerFuncInstance) End() error {
513513
return nil
514514
}
515+
515516
func (ccfi *changeConsumerFuncInstance) Consume(ctx context.Context, change Change) error {
516517
return ccfi.f(ctx, ccfi.tableName, change)
517518
}
@@ -609,7 +610,7 @@ func (crq *changeRowQuerier) queryRange(start, end gocql.UUID) (*changeRowIterat
609610
}
610611

611612
// For a given range, returns the cdc$time of the earliest rows for each stream.
612-
func (crq *changeRowQuerier) findFirstRowsInRange(start, end gocql.UUID) (map[string]gocql.UUID, error) {
613+
func (crq *changeRowQuerier) findFirstRowsInRange(start, end gocql.UUID) (map[string]gocql.UUID, error) { // nolint:unused
613614
queryStr := fmt.Sprintf(
614615
"SELECT \"cdc$stream_id\", \"cdc$time\" FROM %s.%s%s WHERE %s AND \"cdc$time\" > ? AND \"cdc$time\" <= ? PER PARTITION LIMIT 1 BYPASS CACHE",
615616
crq.keyspaceName,
@@ -644,46 +645,46 @@ func (crq *changeRowQuerier) findFirstRowsInRange(start, end gocql.UUID) (map[st
644645
//
645646
// Gocql has two main methods of retrieving row data:
646647
//
647-
// - If you know what columns will be returned by the query and which types
648-
// to use to represent them, you use (*Iter).Scan(...) function and pass
649-
// a list of pointers to values of types you chose for the representation.
650-
// For example, if `x` is int, `Scan(&x)` will put the value of the column
651-
// directly to the `x` variable, setting it to 0 if the column was null.
652-
// - If you don't know which columns will be returned and what are their
653-
// types, you can use (*Iter).MapScan, which returns a map from column
654-
// name to the column value. Gocql automatically chooses a type which
655-
// will be used to represent the column value.
648+
// - If you know what columns will be returned by the query and which types
649+
// to use to represent them, you use (*Iter).Scan(...) function and pass
650+
// a list of pointers to values of types you chose for the representation.
651+
// For example, if `x` is int, `Scan(&x)` will put the value of the column
652+
// directly to the `x` variable, setting it to 0 if the column was null.
653+
// - If you don't know which columns will be returned and what are their
654+
// types, you can use (*Iter).MapScan, which returns a map from column
655+
// name to the column value. Gocql automatically chooses a type which
656+
// will be used to represent the column value.
656657
//
657658
// In our interface, we would like to use an API like MapScan, but there
658659
// are some problems which are addressed by changeRowIterator:
659660
//
660-
// - Gocql's choice of the type used to represent column values is not the best
661-
// for CDC use case. First and foremost, it's very important to differentiate
662-
// Go's default value for a type from a null. For example, for int columns,
663-
// MapScan chooses Go's int type, and sets it to 0 in both cases if it was 0
664-
// or null in the table. For CDC, this means completely different things -
665-
// 0 would mean that the 0 value was written to that column, while null would
666-
// mean that this column value was not changed.
667-
// Fortunately, we can solve this issue by using a pointer-to-type (e.g. *int).
668-
// Gocql will set it to null if it was null in the database, and set it
669-
// to a pointer to a proper value if it was not null.
661+
// - Gocql's choice of the type used to represent column values is not the best
662+
// for CDC use case. First and foremost, it's very important to differentiate
663+
// Go's default value for a type from a null. For example, for int columns,
664+
// MapScan chooses Go's int type, and sets it to 0 in both cases if it was 0
665+
// or null in the table. For CDC, this means completely different things -
666+
// 0 would mean that the 0 value was written to that column, while null would
667+
// mean that this column value was not changed.
668+
// Fortunately, we can solve this issue by using a pointer-to-type (e.g. *int).
669+
// Gocql will set it to null if it was null in the database, and set it
670+
// to a pointer to a proper value if it was not null.
670671
//
671-
// - Similarly to above, UDTs suffer from a similar problem - they are,
672-
// by default, represented by a map[string]interface{} which holds non-pointer
673-
// values of UDT's elements. Fortunately, we can provide a custom type
674-
// which uses pointers to UDT's elements - see udtWithNulls.
672+
// - Similarly to above, UDTs suffer from a similar problem - they are,
673+
// by default, represented by a map[string]interface{} which holds non-pointer
674+
// values of UDT's elements. Fortunately, we can provide a custom type
675+
// which uses pointers to UDT's elements - see udtWithNulls.
675676
//
676-
// - Tuples are handled in a peculiar way - instead of returning, for example,
677-
// an []interface{} which holds tuple values, Scan expects that a pointer
678-
// for each tuple element will be provided, and MapScan puts each tuple
679-
// element under a separate key in the map. This creates a problem - it's
680-
// impossible to differentiate a tuple with all fields set to null, and
681-
// a tuple that is just a null. In CDC, the first means an overwrite of the
682-
// column, and the second means that the column should not be changed.
683-
// This is worked around by using the writetime(X) function on the tuple
684-
// column - this function returns null iff column X was null.
685-
// Moreover, tuples are represented as an []interface{} slice containing
686-
// pointers to tuple elements.
677+
// - Tuples are handled in a peculiar way - instead of returning, for example,
678+
// an []interface{} which holds tuple values, Scan expects that a pointer
679+
// for each tuple element will be provided, and MapScan puts each tuple
680+
// element under a separate key in the map. This creates a problem - it's
681+
// impossible to differentiate a tuple with all fields set to null, and
682+
// a tuple that is just a null. In CDC, the first means an overwrite of the
683+
// column, and the second means that the column should not be changed.
684+
// This is worked around by using the writetime(X) function on the tuple
685+
// column - this function returns null iff column X was null.
686+
// Moreover, tuples are represented as an []interface{} slice containing
687+
// pointers to tuple elements.
687688
type changeRowIterator struct {
688689
iter *gocql.Iter
689690
columnValues []interface{}

consumer_test.go

+17-18
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package scyllacdc
1+
package scyllacdc_test
22

33
import (
44
"context"
@@ -9,6 +9,8 @@ import (
99
"time"
1010

1111
"github.com/gocql/gocql"
12+
13+
scyllacdc "github.com/scylladb/scylla-cdc-go"
1214
"github.com/scylladb/scylla-cdc-go/internal/testutils"
1315
)
1416

@@ -17,14 +19,11 @@ type recordingConsumer struct {
1719
emptyTimestamps []gocql.UUID
1820
}
1921

20-
func (rc *recordingConsumer) CreateChangeConsumer(
21-
ctx context.Context,
22-
input CreateChangeConsumerInput,
23-
) (ChangeConsumer, error) {
22+
func (rc *recordingConsumer) CreateChangeConsumer(_ context.Context, _ scyllacdc.CreateChangeConsumerInput) (scyllacdc.ChangeConsumer, error) {
2423
return rc, nil
2524
}
2625

27-
func (rc *recordingConsumer) Consume(ctx context.Context, change Change) error {
26+
func (rc *recordingConsumer) Consume(ctx context.Context, change scyllacdc.Change) error {
2827
return nil
2928
}
3029

@@ -49,7 +48,7 @@ func (rc *recordingConsumer) GetTimestamps() []gocql.UUID {
4948
func TestConsumerCallsEmptyCallback(t *testing.T) {
5049
consumer := &recordingConsumer{mu: &sync.Mutex{}}
5150

52-
adv := AdvancedReaderConfig{
51+
adv := scyllacdc.AdvancedReaderConfig{
5352
ChangeAgeLimit: -time.Millisecond,
5453
PostNonEmptyQueryDelay: 100 * time.Millisecond,
5554
PostEmptyQueryDelay: 100 * time.Millisecond,
@@ -72,7 +71,7 @@ func TestConsumerCallsEmptyCallback(t *testing.T) {
7271

7372
execQuery(t, session, "CREATE TABLE tbl (pk int PRIMARY KEY, v int) WITH cdc = {'enabled': true}")
7473

75-
cfg := &ReaderConfig{
74+
cfg := &scyllacdc.ReaderConfig{
7675
Session: session,
7776
ChangeConsumerFactory: consumer,
7877
TableNames: []string{keyspaceName + ".tbl"},
@@ -82,7 +81,7 @@ func TestConsumerCallsEmptyCallback(t *testing.T) {
8281

8382
startTime := time.Now()
8483

85-
reader, err := NewReader(context.Background(), cfg)
84+
reader, err := scyllacdc.NewReader(context.Background(), cfg)
8685
if err != nil {
8786
t.Fatal(err)
8887
}
@@ -139,13 +138,13 @@ func TestConsumerResumesWithTableBackedProgressReporter(t *testing.T) {
139138

140139
execQuery(t, session, "CREATE TABLE tbl (pk int PRIMARY KEY, v int) WITH cdc = {'enabled': true}")
141140

142-
runWithProgressReporter := func(consumerFactory ChangeConsumerFactory, endTime time.Time, adv AdvancedReaderConfig) {
143-
progressManager, err := NewTableBackedProgressManager(session, "progress", "test")
141+
runWithProgressReporter := func(consumerFactory scyllacdc.ChangeConsumerFactory, endTime time.Time, adv scyllacdc.AdvancedReaderConfig) {
142+
progressManager, err := scyllacdc.NewTableBackedProgressManager(session, "progress", "test")
144143
if err != nil {
145144
t.Fatalf("failed to create progress manager: %v", err)
146145
}
147146

148-
cfg := &ReaderConfig{
147+
cfg := &scyllacdc.ReaderConfig{
149148
Session: session,
150149
ChangeConsumerFactory: consumerFactory,
151150
TableNames: []string{keyspaceName + ".tbl"},
@@ -154,7 +153,7 @@ func TestConsumerResumesWithTableBackedProgressReporter(t *testing.T) {
154153
Logger: log.New(os.Stderr, "", log.Ldate|log.Lmicroseconds|log.Lshortfile),
155154
}
156155

157-
reader, err := NewReader(context.Background(), cfg)
156+
reader, err := scyllacdc.NewReader(context.Background(), cfg)
158157
if err != nil {
159158
t.Fatal(err)
160159
}
@@ -172,7 +171,7 @@ func TestConsumerResumesWithTableBackedProgressReporter(t *testing.T) {
172171

173172
startTime := time.Now()
174173

175-
adv := AdvancedReaderConfig{
174+
adv := scyllacdc.AdvancedReaderConfig{
176175
PostNonEmptyQueryDelay: 100 * time.Millisecond,
177176
PostEmptyQueryDelay: 100 * time.Millisecond,
178177
PostFailedQueryDelay: 100 * time.Millisecond,
@@ -239,26 +238,26 @@ func TestConsumerHonorsTableTTL(t *testing.T) {
239238
startTime := time.Now()
240239
endTime := startTime.Add(2 * time.Second)
241240

242-
adv := AdvancedReaderConfig{
241+
adv := scyllacdc.AdvancedReaderConfig{
243242
PostNonEmptyQueryDelay: 100 * time.Millisecond,
244243
PostEmptyQueryDelay: 100 * time.Millisecond,
245244
PostFailedQueryDelay: 100 * time.Millisecond,
246245
QueryTimeWindowSize: 500 * time.Millisecond,
247246
ConfidenceWindowSize: time.Millisecond,
248-
ChangeAgeLimit: time.Minute, // should be overriden by the TTL
247+
ChangeAgeLimit: time.Minute, // should be overridden by the TTL
249248
}
250249

251250
consumer := &recordingConsumer{mu: &sync.Mutex{}}
252251

253-
cfg := &ReaderConfig{
252+
cfg := &scyllacdc.ReaderConfig{
254253
Session: session,
255254
ChangeConsumerFactory: consumer,
256255
TableNames: []string{keyspaceName + ".tbl"},
257256
Advanced: adv,
258257
Logger: log.New(os.Stderr, "", log.Ldate|log.Lmicroseconds|log.Lshortfile),
259258
}
260259

261-
reader, err := NewReader(context.Background(), cfg)
260+
reader, err := scyllacdc.NewReader(context.Background(), cfg)
262261
if err != nil {
263262
t.Fatal(err)
264263
}

doc.go

+4-5
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@ It is recommended to get familiar with the Scylla CDC documentation first
66
in order to understand the concepts used in the documentation of scyllacdc:
77
https://docs.scylladb.com/using-scylla/cdc/
88
9-
Overview
9+
# Overview
1010
1111
The library hides the complexity of reading from CDC log stemming from
1212
the need for polling for changes and handling topology changes. It reads
1313
changes from CDC logs of selected tables and propagates them to instances
1414
of ChangeConsumer - which is an interface that is meant to be implemented
1515
by the user.
1616
17-
Getting started
17+
# Getting started
1818
1919
To start working with the library, you first need to implement your own
2020
logic for consuming changes. The simplest way to do it is to define
@@ -95,7 +95,7 @@ Next, you need to create and run a scyllacdc.Reader object:
9595
}
9696
}
9797
98-
Saving progress
98+
# Saving progress
9999
100100
The library supports saving progress and restoring from the last saved position.
101101
To enable it, you need to do two things:
@@ -147,7 +147,7 @@ In the main function:
147147
148148
cfg.ProgressReporter = scyllacdc.NewTableBackedProgressManager("my_keyspace.progress_table", "my_application_name")
149149
150-
Processing changes
150+
# Processing changes
151151
152152
Data from the CDC log is supplied to the ChangeConsumer through Change objects,
153153
which can contain multiple ChangeRow objects. A single ChangeRow corresponds
@@ -179,6 +179,5 @@ to a single, full (all columns included) row from the CDC log.
179179
180180
return nil
181181
}
182-
183182
*/
184183
package scyllacdc

0 commit comments

Comments
 (0)