Skip to content

Commit 5df8640

Browse files
HTHouHTHou
committed
Support nil in Tablet
1 parent c8eaf12 commit 5df8640

File tree

5 files changed

+250
-7
lines changed

5 files changed

+250
-7
lines changed

‎client/bitmap.go

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package client
21+
22+
type BitMap struct {
23+
size int
24+
bits []byte
25+
}
26+
27+
var BitUtil = []byte{1, 2, 4, 8, 16, 32, 64, 128}
28+
29+
func NewBitMap(size int) *BitMap {
30+
bitMap := &BitMap{
31+
size: size,
32+
bits: make([]byte, (size+7)/8),
33+
}
34+
return bitMap
35+
}
36+
37+
func (b *BitMap) Mark(position int) {
38+
b.bits[position/8] |= BitUtil[position%8]
39+
}
40+
41+
func (b *BitMap) IsMarked(position int) bool {
42+
return (b.bits[position/8] & BitUtil[position%8]) != 0
43+
}
44+
45+
func (b *BitMap) IsAllUnmarked() bool {
46+
for i := 0; i < b.size/8; i++ {
47+
if b.bits[i] != 0 {
48+
return false
49+
}
50+
}
51+
for i := 0; i < b.size%8; i++ {
52+
if (b.bits[b.size/8] & BitUtil[i]) != 0 {
53+
return false
54+
}
55+
}
56+
return true
57+
}
58+
59+
func (b *BitMap) GetBits() []byte {
60+
return b.bits
61+
}

‎client/tablet.go

+28-6
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ package client
2222
import (
2323
"bytes"
2424
"encoding/binary"
25-
"errors"
2625
"fmt"
2726
"reflect"
2827
"sort"
@@ -38,6 +37,7 @@ type Tablet struct {
3837
measurementSchemas []*MeasurementSchema
3938
timestamps []int64
4039
values []interface{}
40+
bitMaps []*BitMap
4141
maxRowNumber int
4242
RowSize int
4343
}
@@ -81,18 +81,27 @@ func (t *Tablet) SetTimestamp(timestamp int64, rowIndex int) {
8181
}
8282

8383
func (t *Tablet) SetValueAt(value interface{}, columnIndex, rowIndex int) error {
84-
if value == nil {
85-
return errors.New("illegal argument value can't be nil")
86-
}
8784

8885
if columnIndex < 0 || columnIndex > len(t.measurementSchemas) {
8986
return fmt.Errorf("illegal argument columnIndex %d", columnIndex)
9087
}
9188

92-
if rowIndex < 0 || rowIndex > int(t.maxRowNumber) {
89+
if rowIndex < 0 || rowIndex > t.maxRowNumber {
9390
return fmt.Errorf("illegal argument rowIndex %d", rowIndex)
9491
}
9592

93+
if value == nil {
94+
// Init the bitMap to mark nil value
95+
if t.bitMaps == nil {
96+
t.bitMaps = make([]*BitMap, len(t.values))
97+
}
98+
if t.bitMaps[columnIndex] == nil {
99+
t.bitMaps[columnIndex] = NewBitMap(t.maxRowNumber)
100+
}
101+
// Mark the nil value position
102+
t.bitMaps[columnIndex].Mark(rowIndex)
103+
}
104+
96105
switch t.measurementSchemas[columnIndex].DataType {
97106
case BOOLEAN:
98107
values := t.values[columnIndex].([]bool)
@@ -167,11 +176,15 @@ func (t *Tablet) GetValueAt(columnIndex, rowIndex int) (interface{}, error) {
167176
return nil, fmt.Errorf("illegal argument columnIndex %d", columnIndex)
168177
}
169178

170-
if rowIndex < 0 || rowIndex > int(t.maxRowNumber) {
179+
if rowIndex < 0 || rowIndex > t.maxRowNumber {
171180
return nil, fmt.Errorf("illegal argument rowIndex %d", rowIndex)
172181
}
173182

174183
schema := t.measurementSchemas[columnIndex]
184+
185+
if t.bitMaps != nil && t.bitMaps[columnIndex] != nil && t.bitMaps[columnIndex].IsMarked(rowIndex) {
186+
return nil, nil
187+
}
175188
switch schema.DataType {
176189
case BOOLEAN:
177190
return t.values[columnIndex].([]bool)[rowIndex], nil
@@ -235,6 +248,15 @@ func (t *Tablet) getValuesBytes() ([]byte, error) {
235248
return nil, fmt.Errorf("illegal datatype %v", schema.DataType)
236249
}
237250
}
251+
if t.bitMaps != nil {
252+
for _, bitMap := range t.bitMaps {
253+
columnHasNil := bitMap != nil && !bitMap.IsAllUnmarked()
254+
binary.Write(buff, binary.BigEndian, columnHasNil)
255+
if columnHasNil {
256+
binary.Write(buff, binary.BigEndian, bitMap.GetBits()[0:t.RowSize/8+1])
257+
}
258+
}
259+
}
238260
return buff.Bytes(), nil
239261
}
240262

‎client/tablet_test.go

+83
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,89 @@ func TestTablet_GetValueAt(t *testing.T) {
342342
}
343343
}
344344

345+
func TestTablet_GetNilValueAt(t *testing.T) {
346+
type args struct {
347+
columnIndex int
348+
rowIndex int
349+
}
350+
tests := []struct {
351+
name string
352+
args args
353+
want interface{}
354+
wantErr bool
355+
}{
356+
{
357+
name: "INT32",
358+
args: args{
359+
columnIndex: 0,
360+
rowIndex: 0,
361+
},
362+
want: int32(256),
363+
wantErr: false,
364+
}, {
365+
name: "FLOAT64",
366+
args: args{
367+
columnIndex: 1,
368+
rowIndex: 0,
369+
},
370+
want: nil,
371+
wantErr: false,
372+
}, {
373+
name: "INT64",
374+
args: args{
375+
columnIndex: 2,
376+
rowIndex: 0,
377+
},
378+
want: int64(65535),
379+
wantErr: false,
380+
}, {
381+
name: "FLOAT32",
382+
args: args{
383+
columnIndex: 3,
384+
rowIndex: 0,
385+
},
386+
want: float32(36.5),
387+
wantErr: false,
388+
}, {
389+
name: "STRING",
390+
args: args{
391+
columnIndex: 4,
392+
rowIndex: 0,
393+
},
394+
want: "Hello World!",
395+
wantErr: false,
396+
}, {
397+
name: "BOOLEAN",
398+
args: args{
399+
columnIndex: 5,
400+
rowIndex: 0,
401+
},
402+
want: true,
403+
wantErr: false,
404+
},
405+
}
406+
if tablet, err := createTablet(1); err == nil {
407+
tablet.SetValueAt(int32(256), 0, 0)
408+
tablet.SetValueAt(nil, 1, 0)
409+
tablet.SetValueAt(int64(65535), 2, 0)
410+
tablet.SetValueAt(float32(36.5), 3, 0)
411+
tablet.SetValueAt("Hello World!", 4, 0)
412+
tablet.SetValueAt(true, 5, 0)
413+
for _, tt := range tests {
414+
t.Run(tt.name, func(t *testing.T) {
415+
got, err := tablet.GetValueAt(tt.args.columnIndex, tt.args.rowIndex)
416+
if (err != nil) != tt.wantErr {
417+
t.Errorf("Tablet.GetValueAt() error = %v, wantErr %v", err, tt.wantErr)
418+
return
419+
}
420+
if !reflect.DeepEqual(got, tt.want) {
421+
t.Errorf("Tablet.GetValueAt() = %v, want %v", got, tt.want)
422+
}
423+
})
424+
}
425+
}
426+
}
427+
345428
func TestTablet_Sort(t *testing.T) {
346429

347430
tests := []struct {

‎example/session_example.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,11 @@ func createTablet(rowCount int) (*client.Tablet, error) {
531531
ts++
532532
tablet.SetTimestamp(ts, row)
533533
tablet.SetValueAt(rand.Int31(), 0, row)
534-
tablet.SetValueAt(rand.Float64(), 1, row)
534+
if row%2 == 1 {
535+
tablet.SetValueAt(rand.Float64(), 1, row)
536+
} else {
537+
tablet.SetValueAt(nil, 1, row)
538+
}
535539
tablet.SetValueAt(rand.Int63(), 2, row)
536540
tablet.SetValueAt(rand.Float32(), 3, row)
537541
tablet.SetValueAt(fmt.Sprintf("Test Device %d", row+1), 4, row)

‎test/e2e/e2e_test.go

+73
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,79 @@ func (s *e2eTestSuite) Test_InsertAlignedTablet() {
236236
assert.Equal(status, "12")
237237
s.session.DeleteStorageGroup("root.ln.**")
238238
}
239+
240+
func (s *e2eTestSuite) Test_InsertAlignedTabletWithNilValue() {
241+
var timeseries = []string{"root.ln.device1.**"}
242+
s.session.DeleteTimeseries(timeseries)
243+
if tablet, err := createTabletWithNil(12); err == nil {
244+
status, err := s.session.InsertAlignedTablet(tablet, false)
245+
s.checkError(status, err)
246+
tablet.Reset()
247+
} else {
248+
log.Fatal(err)
249+
}
250+
var timeout int64 = 1000
251+
ds, err := s.session.ExecuteQueryStatement("select count(status) from root.ln.device1", &timeout)
252+
assert := s.Require()
253+
assert.NoError(err)
254+
defer ds.Close()
255+
assert.True(ds.Next())
256+
var status string
257+
assert.NoError(ds.Scan(&status))
258+
assert.Equal(status, "12")
259+
s.session.DeleteStorageGroup("root.ln.**")
260+
}
261+
262+
func createTabletWithNil(rowCount int) (*client.Tablet, error) {
263+
tablet, err := client.NewTablet("root.ln.device1", []*client.MeasurementSchema{
264+
{
265+
Measurement: "restart_count",
266+
DataType: client.INT32,
267+
}, {
268+
Measurement: "price",
269+
DataType: client.DOUBLE,
270+
}, {
271+
Measurement: "tick_count",
272+
DataType: client.INT64,
273+
}, {
274+
Measurement: "temperature",
275+
DataType: client.FLOAT,
276+
}, {
277+
Measurement: "description",
278+
DataType: client.TEXT,
279+
},
280+
{
281+
Measurement: "status",
282+
DataType: client.BOOLEAN,
283+
},
284+
}, rowCount)
285+
286+
if err != nil {
287+
return nil, err
288+
}
289+
ts := time.Now().UTC().UnixNano() / 1000000
290+
for row := 0; row < int(rowCount); row++ {
291+
ts++
292+
tablet.SetTimestamp(ts, row)
293+
tablet.SetValueAt(rand.Int31(), 0, row)
294+
if row%2 == 1 {
295+
tablet.SetValueAt(rand.Float64(), 1, row)
296+
} else {
297+
tablet.SetValueAt(nil, 1, row)
298+
}
299+
tablet.SetValueAt(rand.Int63(), 2, row)
300+
if row%3 == 1 {
301+
tablet.SetValueAt(rand.Float32(), 3, row)
302+
} else {
303+
tablet.SetValueAt(nil, 3, row)
304+
}
305+
tablet.SetValueAt(fmt.Sprintf("Test Device %d", row+1), 4, row)
306+
tablet.SetValueAt(bool(ts%2 == 0), 5, row)
307+
tablet.RowSize++
308+
}
309+
return tablet, nil
310+
}
311+
239312
func createTablet(rowCount int) (*client.Tablet, error) {
240313
tablet, err := client.NewTablet("root.ln.device1", []*client.MeasurementSchema{
241314
{

0 commit comments

Comments
 (0)