Skip to content

Commit 5435f02

Browse files
SimFGSimFG
committed
cdc with ts
1 parent 832468c commit 5435f02

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+6634
-5147
lines changed

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,5 @@
77
**/__pycache__/*
88
**/.pytest_cache/*
99
server/cdc
10-
**/milvus-cdc-build/*
10+
**/milvus-cdc-build/*
11+
*.log

‎Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,9 @@ test-go:
77
static-check:
88
@echo "Running go-lint check:"
99
@(env bash $(PWD)/scripts/run_go_lint.sh)
10+
11+
# TODO use the array to generate the name list
12+
generate-mockery:
13+
@echo "Generating mockery server mocks..."
14+
@cd "$(PWD)/server"; mockery -r --name "CDCService|CDCFactory|MetaStore|MetaStoreFactory" --output ./mocks --case snake --with-expecter
15+
@cd "$(PWD)/core"; mockery -r --name "CDCReader|CDCWriter|FactoryCreator|Monitor|WriteCallback|MilvusClientFactory|MilvusClientAPI|ChannelManager|TargetAPI|MetaOp" --output ./mocks --case snake --with-expecter

‎core/api/data_handler.go

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
package api
2+
3+
import (
4+
"context"
5+
6+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
7+
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
8+
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
9+
"github.com/milvus-io/milvus-sdk-go/v2/entity"
10+
"github.com/milvus-io/milvus/pkg/log"
11+
)
12+
13+
type DataHandler interface {
14+
CreateCollection(ctx context.Context, param *CreateCollectionParam) error
15+
DropCollection(ctx context.Context, param *DropCollectionParam) error
16+
Insert(ctx context.Context, param *InsertParam) error
17+
Delete(ctx context.Context, param *DeleteParam) error
18+
CreatePartition(ctx context.Context, param *CreatePartitionParam) error
19+
DropPartition(ctx context.Context, param *DropPartitionParam) error
20+
21+
CreateIndex(ctx context.Context, param *CreateIndexParam) error
22+
DropIndex(ctx context.Context, param *DropIndexParam) error
23+
LoadCollection(ctx context.Context, param *LoadCollectionParam) error
24+
ReleaseCollection(ctx context.Context, param *ReleaseCollectionParam) error
25+
CreateDatabase(ctx context.Context, param *CreateDataBaseParam) error
26+
DropDatabase(ctx context.Context, param *DropDataBaseParam) error
27+
28+
ReplicateMessage(ctx context.Context, param *ReplicateMessageParam) error
29+
// NOTE: please add the implements for the DataHandlerWrapper class when adding new interfaces
30+
}
31+
32+
type DefaultDataHandler struct{}
33+
34+
func (d *DefaultDataHandler) CreateCollection(ctx context.Context, param *CreateCollectionParam) error {
35+
log.Warn("CreateCollection is not implemented, please check it")
36+
return nil
37+
}
38+
39+
func (d *DefaultDataHandler) DropCollection(ctx context.Context, param *DropCollectionParam) error {
40+
log.Warn("DropCollection is not implemented, please check it")
41+
return nil
42+
}
43+
44+
func (d *DefaultDataHandler) Insert(ctx context.Context, param *InsertParam) error {
45+
log.Warn("Insert is not implemented, please check it")
46+
return nil
47+
}
48+
49+
func (d *DefaultDataHandler) Delete(ctx context.Context, param *DeleteParam) error {
50+
log.Warn("Delete is not implemented, please check it")
51+
return nil
52+
}
53+
54+
func (d *DefaultDataHandler) CreatePartition(ctx context.Context, param *CreatePartitionParam) error {
55+
log.Warn("CreatePartition is not implemented, please check it")
56+
return nil
57+
}
58+
59+
func (d *DefaultDataHandler) DropPartition(ctx context.Context, param *DropPartitionParam) error {
60+
log.Warn("DropPartition is not implemented, please check it")
61+
return nil
62+
}
63+
64+
func (d *DefaultDataHandler) CreateIndex(ctx context.Context, param *CreateIndexParam) error {
65+
log.Warn("CreateIndex is not implemented, please check it")
66+
return nil
67+
}
68+
69+
func (d *DefaultDataHandler) DropIndex(ctx context.Context, param *DropIndexParam) error {
70+
log.Warn("DropIndex is not implemented, please check it")
71+
return nil
72+
}
73+
74+
func (d *DefaultDataHandler) LoadCollection(ctx context.Context, param *LoadCollectionParam) error {
75+
log.Warn("LoadCollection is not implemented, please check it")
76+
return nil
77+
}
78+
79+
func (d *DefaultDataHandler) ReleaseCollection(ctx context.Context, param *ReleaseCollectionParam) error {
80+
log.Warn("ReleaseCollection is not implemented, please check it")
81+
return nil
82+
}
83+
84+
func (d *DefaultDataHandler) CreateDatabase(ctx context.Context, param *CreateDataBaseParam) error {
85+
log.Warn("CreateDatabase is not implemented, please check it")
86+
return nil
87+
}
88+
89+
func (d *DefaultDataHandler) DropDatabase(ctx context.Context, param *DropDataBaseParam) error {
90+
log.Warn("DropDatabase is not implemented, please check it")
91+
return nil
92+
}
93+
94+
func (d *DefaultDataHandler) ReplicateMessage(ctx context.Context, param *ReplicateMessageParam) error {
95+
log.Warn("Replicate is not implemented, please check it")
96+
return nil
97+
}
98+
99+
type CreateCollectionParam struct {
100+
Schema *entity.Schema
101+
ShardsNum int32
102+
ConsistencyLevel commonpb.ConsistencyLevel
103+
Properties []*commonpb.KeyValuePair
104+
}
105+
106+
type DropCollectionParam struct {
107+
CollectionName string
108+
}
109+
110+
type InsertParam struct {
111+
CollectionName string
112+
PartitionName string
113+
Columns []entity.Column
114+
}
115+
116+
type DeleteParam struct {
117+
CollectionName string
118+
PartitionName string
119+
Column entity.Column
120+
}
121+
122+
type CreatePartitionParam struct {
123+
CollectionName string
124+
PartitionName string
125+
}
126+
127+
type DropPartitionParam struct {
128+
CollectionName string
129+
PartitionName string
130+
}
131+
132+
type CreateIndexParam struct {
133+
milvuspb.CreateIndexRequest
134+
}
135+
136+
type DropIndexParam struct {
137+
milvuspb.DropIndexRequest
138+
}
139+
140+
type LoadCollectionParam struct {
141+
milvuspb.LoadCollectionRequest
142+
}
143+
144+
type ReleaseCollectionParam struct {
145+
milvuspb.ReleaseCollectionRequest
146+
}
147+
148+
type CreateDataBaseParam struct {
149+
milvuspb.CreateDatabaseRequest
150+
}
151+
152+
type DropDataBaseParam struct {
153+
milvuspb.DropDatabaseRequest
154+
}
155+
156+
type ReplicateMessageParam struct {
157+
ChannelName string
158+
BeginTs, EndTs uint64
159+
MsgsBytes [][]byte
160+
StartPositions, EndPositions []*msgpb.MsgPosition
161+
}

‎core/api/message_manager.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package api
2+
3+
type MessageManager interface {
4+
ReplicateMessage(message *ReplicateMessage)
5+
Close(channelName string)
6+
}
7+
8+
type ReplicateMessage struct {
9+
Param *ReplicateMessageParam
10+
SuccessFunc func(param *ReplicateMessageParam)
11+
FailFunc func(param *ReplicateMessageParam, err error)
12+
}

‎core/api/meta_op.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package api
2+
3+
import (
4+
"context"
5+
6+
"github.com/zilliztech/milvus-cdc/core/pb"
7+
)
8+
9+
// MetaOp meta operation
10+
type MetaOp interface {
11+
// WatchCollection its implementation should make sure it's only called once. The WatchPartition is same
12+
WatchCollection(ctx context.Context, filter CollectionFilter)
13+
WatchPartition(ctx context.Context, filter PartitionFilter)
14+
15+
// SubscribeCollectionEvent an event only is consumed once. The SubscribePartitionEvent is same
16+
// TODO need to consider the many target, maybe try the method a meta op corresponds to a target
17+
SubscribeCollectionEvent(taskID string, consumer CollectionEventConsumer)
18+
SubscribePartitionEvent(taskID string, consumer PartitionEventConsumer)
19+
UnsubscribeEvent(taskID string, eventType WatchEventType)
20+
21+
GetAllCollection(ctx context.Context, filter CollectionFilter) ([]*pb.CollectionInfo, error)
22+
GetCollectionNameByID(ctx context.Context, id int64) string
23+
}
24+
25+
// CollectionFilter the filter will be used before the collection is filled the schema info
26+
type CollectionFilter func(*pb.CollectionInfo) bool
27+
28+
type PartitionFilter func(info *pb.PartitionInfo) bool
29+
30+
type CollectionEventConsumer CollectionFilter
31+
32+
type PartitionEventConsumer PartitionFilter
33+
34+
type WatchEventType int
35+
36+
const (
37+
CollectionEventType WatchEventType = iota + 1
38+
PartitionEventType
39+
)

‎core/api/reader.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package api
2+
3+
import (
4+
"context"
5+
6+
"github.com/milvus-io/milvus/pkg/log"
7+
)
8+
9+
type Reader interface {
10+
StartRead(ctx context.Context)
11+
QuitRead(ctx context.Context)
12+
GetChannelChan() <-chan string
13+
}
14+
15+
// DefaultReader All CDCReader implements should combine it
16+
type DefaultReader struct{}
17+
18+
// StartRead the return value is nil,
19+
// and if you receive the data from the nil chan, will block forever, not panic
20+
func (d *DefaultReader) StartRead(ctx context.Context) {
21+
log.Warn("StartRead is not implemented, please check it")
22+
}
23+
24+
func (d *DefaultReader) QuitRead(ctx context.Context) {
25+
log.Warn("QuitRead is not implemented, please check it")
26+
}

‎core/api/replicate_manager.go

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package api
2+
3+
import (
4+
"context"
5+
6+
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
7+
"github.com/milvus-io/milvus/pkg/mq/msgstream"
8+
9+
"github.com/zilliztech/milvus-cdc/core/model"
10+
"github.com/zilliztech/milvus-cdc/core/pb"
11+
)
12+
13+
// ChannelManager a target must promise a manager
14+
type ChannelManager interface {
15+
StartReadCollection(ctx context.Context, info *pb.CollectionInfo, seekPositions []*msgpb.MsgPosition) error
16+
17+
StopReadCollection(ctx context.Context, info *pb.CollectionInfo) error
18+
19+
GetChannelChan() <-chan string
20+
GetMsgChan(pChannel string) <-chan *msgstream.MsgPack
21+
GetEventChan() <-chan *ReplicateAPIEvent
22+
}
23+
24+
type TargetAPI interface {
25+
GetCollectionInfo(ctx context.Context, collectionName string) (*model.CollectionInfo, error)
26+
}
27+
28+
type ReplicateAPIEvent struct {
29+
EventType ReplicateAPIEventType
30+
CollectionInfo *pb.CollectionInfo
31+
PartitionInfo *pb.PartitionInfo
32+
}
33+
34+
type ReplicateAPIEventType int
35+
36+
const (
37+
ReplicateCreateCollection = iota + 1
38+
ReplicateDropCollection
39+
ReplicateCreatePartition
40+
ReplicateDropPartition
41+
)

‎core/api/writer.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package api
2+
3+
import (
4+
"context"
5+
6+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
7+
"github.com/milvus-io/milvus/pkg/mq/msgstream"
8+
)
9+
10+
type Writer interface {
11+
HandleReplicateAPIEvent(ctx context.Context, apiEvent *ReplicateAPIEvent) error
12+
HandleReplicateMessage(ctx context.Context, channelName string, msgPack *msgstream.MsgPack) (*commonpb.KeyDataPair, error)
13+
}

‎core/config/mq.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ func NewParamGroup() paramtable.ParamGroup {
4444
return group
4545
}
4646

47+
type MQConfig struct {
48+
Pulsar PulsarConfig
49+
Kafka KafkaConfig
50+
}
51+
4752
type KafkaConfig struct {
4853
Address string
4954
}

‎core/go.mod

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,15 @@ go 1.18
44

55
require (
66
github.com/cockroachdb/errors v1.9.1
7-
github.com/goccy/go-json v0.10.2
87
github.com/golang/protobuf v1.5.3
9-
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.1-0.20230911111453-720fcfb1a048
8+
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20230919092633-6ef446ad2aab
109
github.com/milvus-io/milvus-sdk-go/v2 v2.2.1-0.20230814034926-dd5a31f64225
1110
github.com/milvus-io/milvus/pkg v0.0.2-0.20230823021022-7af0f7d90cee
1211
github.com/samber/lo v1.27.0
1312
github.com/stretchr/testify v1.8.3
14-
go.etcd.io/etcd/api/v3 v3.5.5
1513
go.etcd.io/etcd/client/v3 v3.5.5
1614
go.uber.org/zap v1.20.0
15+
sigs.k8s.io/yaml v1.2.0
1716
)
1817

1918
require (
@@ -104,6 +103,7 @@ require (
104103
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
105104
github.com/yusufpapurcu/wmi v1.2.2 // indirect
106105
go.etcd.io/bbolt v1.3.6 // indirect
106+
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
107107
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
108108
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
109109
go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
@@ -137,13 +137,12 @@ require (
137137
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
138138
gopkg.in/yaml.v2 v2.4.0 // indirect
139139
gopkg.in/yaml.v3 v3.0.1 // indirect
140-
sigs.k8s.io/yaml v1.2.0 // indirect
141140
)
142141

143142
replace (
144143
github.com/apache/pulsar-client-go => github.com/milvus-io/pulsar-client-go v0.6.10
145-
github.com/milvus-io/milvus-sdk-go/v2 => github.com/SimFG/milvus-sdk-go/v2 v2.0.0-20230918025012-e4fb30541113
146-
github.com/milvus-io/milvus/pkg => github.com/SimFG/milvus/pkg v0.0.0-20230915085959-b1bd79e12920
144+
github.com/milvus-io/milvus-sdk-go/v2 => github.com/SimFG/milvus-sdk-go/v2 v2.0.0-20230919094145-06acf1ab753c
145+
github.com/milvus-io/milvus/pkg => github.com/SimFG/milvus/pkg v0.0.0-20230925083123-8c1cd0c4b615
147146
github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1
148147
github.com/tecbot/gorocksdb => ./../rocksdb
149148
)

0 commit comments

Comments
 (0)