improvement: [storage] use buffer for reads

We now use buffer for reads to make sure that
if an import/read operation fails, we wont be
left with an empty data array

Before we would dump the data array and then read
the file into it. If the file was corrupt, the operation
would fail and leave as with an empty data array

Now we keep the current data array and dump only
if the import/read operation has finished successfully

Author: Christos Kotsis

db/state.go

@@ -13,16 +13,17 @@ const (
 
 // state struct used by dby storage
 type state struct {
-	data, buffer []interface{}
-	lib          map[string]int
-	ad           int
+	data   []interface{}
+	buffer []*interface{}
+	lib    map[string]int
+	ad     int
 }
 
 // newStateFactory for creating a new v3 State
 func newStateFactory() *state {
 	s := state{
 		data:   make([]interface{}, 0),
-		buffer: make([]interface{}, 0),
+		buffer: make([]*interface{}, 0),
 		lib:    make(map[string]int),
 	}
 	return &s
@@ -33,7 +34,7 @@ func (c *state) Clear() {
 	c.data, c.buffer, c.lib = nil, nil, nil
 
 	c.data = make([]interface{}, 0)
-	c.buffer = make([]interface{}, 0)
+	c.buffer = make([]*interface{}, 0)
 	c.lib = make(map[string]int)
 }
 
@@ -58,7 +59,7 @@ func (c *state) PushData(d interface{}) {
 
 // PushBuffer for appending data to the buffer array
 func (c *state) PushBuffer(d interface{}) {
-	c.buffer = append(c.buffer, d)
+	c.buffer = append(c.buffer, &d)
 }
 
 // GetAllData returns the data array
@@ -67,7 +68,7 @@ func (c *state) GetAllData() []interface{} {
 }
 
 // GetAllBuffer returns the buffer array
-func (c *state) GetAllBuffer() []interface{} {
+func (c *state) GetAllBuffer() []*interface{} {
 	return c.buffer
 }
 
@@ -100,11 +101,21 @@ func (c *state) SetDataFromIndex(v interface{}, i int) error {
 }
 
 // GetBufferFromIndex returns the i'th element from the buffer array
-func (c *state) GetBufferFromIndex(i int) (interface{}, error) {
-	if err := c.IndexInRange(i); err != nil {
-		return nil, wrapErr(err)
+func (c *state) GetBufferFromIndex(i int) (*interface{}, error) {
+	if len(c.buffer)-1 >= i {
+		return c.buffer[i], nil
 	}
-	return c.data[i], nil
+	return nil, fmt.Errorf(ire)
+}
+
+// SetDataFromIndex sets to input value the i'th element from the data array
+func (c *state) SetBufferFromIndex(v interface{}, i int) error {
+	if len(c.buffer)-1 >= i {
+		c.buffer[i] = &v
+		return nil
+	}
+	return fmt.Errorf(ire)
+
 }
 
 // IndexInRange check if index is within data array range
@@ -173,10 +184,10 @@ func (c *state) DeleteData(i int) error {
 	return nil
 }
 
-// CopyBufferToData for copying buffer array over data array
-func (c *state) CopyBufferToData() {
-	copy(c.data, c.buffer)
-}
+// // CopyBufferToData for copying buffer array over data array
+// func (c *state) CopyBufferToData() {
+// 	copy(c.data, c.buffer)
+// }
 
 // UnsetDataArray for deleting all data. This sets data = nil
 func (c *state) UnsetDataArray() {
@@ -197,7 +208,7 @@ func (c *state) UnsetBufferArray() {
 // DeleteBuffer deletes the data from the buffer array
 func (c *state) DeleteBuffer() {
 	c.UnsetBufferArray()
-	c.buffer = append(c.buffer, 0)
+	c.buffer = make([]*interface{}, 0)
 }
 
 // ClearLib removes all keys from the lib map

db/store_lib.go

@@ -213,43 +213,46 @@ func (s *Storage) ImportDocs(path string, o ...bool) error {
 		return wrapErr(err)
 	}
 
-	var dataArray []interface{}
 	var counter int
 	var data interface{}
+	s.State.UnsetBufferArray()
 
-	if len(o) > 0 {
-		issueWarning(deprecatedFeature, "ImportDocs(string, bool)", "Storage.DeleteAll(true).ImportDocs(path)")
-		if o[0] {
-			s.State.UnsetDataArray()
-		}
-	}
-
-	data = nil
 	dec := yaml.NewDecoder(bytes.NewReader(impf))
 	for {
-		dataArray = append(dataArray, data)
-		err := dec.Decode(&dataArray[counter])
+		err = dec.Decode(&data)
 		if err == nil {
-			counter++
+			s.State.PushBuffer(data)
 			data = nil
+
+			counter++
 			continue
 		}
 
 		if err.Error() == "EOF" {
 			break
 		}
+		s.State.UnsetBufferArray()
 		return wrapErr(err)
 	}
 
-	for _, j := range dataArray {
+	if len(o) > 0 {
+		issueWarning(deprecatedFeature, "ImportDocs(string, bool)", "Storage.DeleteAll(true).ImportDocs(path)")
+		if o[0] {
+			s.State.UnsetDataArray()
+			s.State.ClearLib()
+		}
+	}
+
+	for _, j := range s.State.GetAllBuffer() {
 		if j == nil {
 			continue
 		}
-		if len(j.(map[interface{}]interface{})) == 0 {
+		if len((*j).(map[interface{}]interface{})) == 0 {
 			continue
 		}
-		s.State.PushData(j)
+		s.State.PushData(*j)
 	}
+	s.State.UnsetBufferArray()
 	return s.stateReload()
 }
 
@@ -270,24 +273,34 @@ func (s *Storage) Read() error {
 	s.Lock()
 	defer s.Unlock()
 
-	s.State.DeleteAllData()
+	s.State.UnsetBufferArray()
 
 	var data interface{}
 	dec := yaml.NewDecoder(bytes.NewReader(f))
 	for {
 		err := dec.Decode(&data)
 		if err == nil {
-			s.State.PushData(data)
+			s.State.PushBuffer(data)
 			data = nil
 			continue
 		}
 
 		if err.Error() == "EOF" {
 			break
 		}
+		s.State.UnsetBufferArray()
 		return wrapErr(err)
 	}
 
+	s.State.UnsetDataArray()
+
+	for _, j := range s.State.GetAllBuffer() {
+		if j == nil {
+			continue
+		}
+		s.State.PushData(*j)
+	}
+	s.State.UnsetBufferArray()
 	return nil
 }