Release/v0.12.0 (#216)

* DecodeInt{8, 16} negative limit checks + tests (#125)

Updates #120

Also adds a test for Byteslice encoding and decoding
roundtripping and part slicing then rejoining.

* fix circleci2.0

* some metalinter issues

* skip over default values when encoding time (#178)

* do not encode empty structs, unless `amino:"write_empty"` is set (#179)

* skip over empty structs by default

* slightly more info on panics

* Fix zero time decoding (#190)

- fix decoding of skipped fields in time, or completely skipped time 
- necessary because sec=0 and ns=0 do not result in time.Time{} and vice versa

* Prepare release 0.11.0 (#193)

* Fix time decoding & encoding of arrays and structs

- top-level entry functions called with BinFieldNum:1 to properly encode e.g. arrays of structs (see non-time related test)
- add defaultValue method, different from #196 it deals with multiply nested pointers

* Removed dependency on tmlibs/common

* Always write empty if struct field is pointer

* add tests for #206

- test for pointers to empty struct and nil pointer

* fix proto3 compatibility for empty structs

* Add EmptyStruct to fuzz tests

* Revert "fix proto3 compatibility for empty structs"

* Do not allow encoding of nil struct pointers in a slice/array

* By default, 0-length list elements are decoded as nil

Author: liamsi

.circleci/circle.yml renamed to .circleci/config.yml

@@ -1,12 +1,13 @@
 version: 2
-
 jobs:
   build:
-    working_directory: /go/src/github.com/tendermint/go-amino
     docker:
-    - image: circleci/golang:1.10.3
-      environment:
-        GOBIN: /tmp/workspace/bin
+      - image: circleci/golang:1.10.3
+        environment:
+          GOBIN: /tmp/workspace/bin
+
+    working_directory: /go/src/github.com/tendermint/go-amino
+
     steps:
       - run: mkdir -p /tmp/workspace/bin
       - run: mkdir -p /tmp/workspace/profiles
@@ -17,9 +18,11 @@ jobs:
       - run:
               name: test
               command: |
+                export PATH="$GOBIN:$PATH"
+                go env
                 go version
-                cd $PROJECT_PATH && make get_tools && make get_vendor_deps && make test
-    - save_cache:
-        key: v1-dep-{{ .Branch }}
-        paths:
-        - /go/pkg
+                make get_tools && make get_vendor_deps && make test
+      - save_cache:
+          key: v1-dep-{{ .Branch }}
+          paths:
+            - /go/pkg

CHANGELOG.md

@@ -1,13 +1,35 @@
 # Changelog
 
+## 0.12.0 (August 4, 2018)
+
+BREAKING CHANGE:
+ - Write empty (non-nil) struct pointers, unless (is list element and empty_elements isn't set) #206
+
+## 0.11.1 (July 17, 2018)
+
+IMPROVEMENTS:
+ - Remove dependency on tmlibs/common
+
+## 0.11.0 (June 19, 2018)
+
+BREAKING CHANGE:
+
+ - Do not encode zero values in `EncodeTime`
+ (to match proto3's behaviour) (#178, #190)
+ - Do not encode empty structs, unless explicitly enforced
+ via `amino:"write_empty"` (to match proto3's behaviour) (#179)
+
+IMPROVEMENTS:
+ - DecodeInt{8, 16} negative limit checks (#125)
+
 ## 0.10.1 (June 15, 2018)
 
 FEATURE:
 
  - [aminoscan] aminoscan --color will print ASCII bytes in different colors
- 
+
 BUG FIXES:
- - do not err if prefix bytes are exactly 4 (for registered types) 
+ - do not err if prefix bytes are exactly 4 (for registered types)
 
 ## 0.10.0 (June 12, 2018)
 

Gopkg.lock

@@ -37,10 +37,6 @@
   name = "github.com/stretchr/testify"
   version = "1.2.1"
 
-[[constraint]]
-  name = "github.com/tendermint/tmlibs"
-  version = "0.9.0-rc1"
-
 [prune]
   go-tests = true
   unused-packages = true

Gopkg.toml

@@ -8,14 +8,28 @@ import (
 	"fmt"
 	"io"
 	"reflect"
+	"time"
 )
 
 //----------------------------------------
 // Global methods for global sealed codec.
 var gcdc *Codec
 
+// we use this time to init. a zero value (opposed to reflect.Zero which gives time.Time{} / 01-01-01 00:00:00)
+var zeroTime time.Time
+
+const (
+	unixEpochStr = "1970-01-01 00:00:00 +0000 UTC"
+	epochFmt     = "2006-01-02 15:04:05 +0000 UTC"
+)
+
 func init() {
 	gcdc = NewCodec().Seal()
+	var err error
+	zeroTime, err = time.Parse(epochFmt, unixEpochStr)
+	if err != nil {
+		panic("couldn't parse Zero value for time")
+	}
 }
 
 func MarshalBinary(o interface{}) ([]byte, error) {
@@ -189,7 +203,7 @@ func (cdc *Codec) MarshalBinaryBare(o interface{}) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = cdc.encodeReflectBinary(buf, info, rv, FieldOptions{}, true)
+	err = cdc.encodeReflectBinary(buf, info, rv, FieldOptions{BinFieldNum: 1}, true)
 	if err != nil {
 		return nil, err
 	}
@@ -327,7 +341,7 @@ func (cdc *Codec) UnmarshalBinaryBare(bz []byte, ptr interface{}) error {
 		bz = bz[4:]
 	}
 	// Decode contents into rv.
-	n, err := cdc.decodeReflectBinary(bz, info, rv, FieldOptions{}, true)
+	n, err := cdc.decodeReflectBinary(bz, info, rv, FieldOptions{BinFieldNum: 1}, true)
 	if err != nil {
 		return fmt.Errorf("unmarshal to %v failed after %d bytes (%v): %X", info.Type, n, err, bz)
 	}

amino.go

@@ -401,6 +401,7 @@ func (cdc *Codec) decodeReflectBinaryByteArray(bz []byte, info *TypeInfo, rv ref
 }
 
 // CONTRACT: rv.CanAddr() is true.
+// NOTE: Keep the code structure similar to decodeReflectBinarySlice.
 func (cdc *Codec) decodeReflectBinaryArray(bz []byte, info *TypeInfo, rv reflect.Value, fopts FieldOptions, bare bool) (n int, err error) {
 	if !rv.CanAddr() {
 		panic("rv not addressable")
@@ -461,6 +462,9 @@ func (cdc *Codec) decodeReflectBinaryArray(bz []byte, info *TypeInfo, rv reflect
 			return
 		}
 	} else {
+		// NOTE: ert is for the element value, while einfo.Type is dereferenced.
+		isErtStructPointer := ert.Kind() == reflect.Ptr && einfo.Type.Kind() == reflect.Struct
+
 		// Read elements in unpacked form.
 		for i := 0; i < length; i++ {
 			// Read field key (number and type).
@@ -480,10 +484,16 @@ func (cdc *Codec) decodeReflectBinaryArray(bz []byte, info *TypeInfo, rv reflect
 			}
 			// Decode the next ByteLength bytes into erv.
 			var erv = rv.Index(i)
-			// Special case if next ByteLength bytes are 0x00, set nil.
-			if len(bz) > 0 && bz[0] == 0x00 {
+			// Special case if:
+			//  * next ByteLength bytes are 0x00, and
+			//  * - erv is not a struct pointer, or
+			//    - field option doesn't have EmptyElements set
+			// (the condition below uses demorgan's law)
+			if (len(bz) > 0 && bz[0] == 0x00) &&
+				!(isErtStructPointer && fopts.EmptyElements) {
+
 				slide(&bz, &n, 1)
-				erv.Set(reflect.Zero(erv.Type()))
+				erv.Set(defaultValue(erv.Type()))
 				continue
 			}
 			// Normal case, read next non-nil element from bz.
@@ -547,6 +557,7 @@ func (cdc *Codec) decodeReflectBinaryByteSlice(bz []byte, info *TypeInfo, rv ref
 }
 
 // CONTRACT: rv.CanAddr() is true.
+// NOTE: Keep the code structure similar to decodeReflectBinaryArray.
 func (cdc *Codec) decodeReflectBinarySlice(bz []byte, info *TypeInfo, rv reflect.Value, fopts FieldOptions, bare bool) (n int, err error) {
 	if !rv.CanAddr() {
 		panic("rv not addressable")
@@ -611,6 +622,9 @@ func (cdc *Codec) decodeReflectBinarySlice(bz []byte, info *TypeInfo, rv reflect
 			srv = reflect.Append(srv, erv)
 		}
 	} else {
+		// NOTE: ert is for the element value, while einfo.Type is dereferenced.
+		isErtStructPointer := ert.Kind() == reflect.Ptr && einfo.Type.Kind() == reflect.Struct
+
 		// Read elements in unpacked form.
 		for {
 			if len(bz) == 0 {
@@ -636,10 +650,16 @@ func (cdc *Codec) decodeReflectBinarySlice(bz []byte, info *TypeInfo, rv reflect
 			}
 			// Decode the next ByteLength bytes into erv.
 			erv, _n := reflect.New(ert).Elem(), int(0)
-			// Special case if next ByteLength bytes are 0x00, set nil.
-			if len(bz) > 0 && bz[0] == 0x00 {
+			// Special case if:
+			//  * next ByteLength bytes are 0x00, and
+			//  * - erv is not a struct pointer, or
+			//    - field option doesn't have EmptyElements set
+			// (the condition below uses demorgan's law)
+			if (len(bz) > 0 && bz[0] == 0x00) &&
+				!(isErtStructPointer && fopts.EmptyElements) {
+
 				slide(&bz, &n, 1)
-				erv.Set(reflect.Zero(erv.Type()))
+				erv.Set(defaultValue(erv.Type()))
 				srv = reflect.Append(srv, erv)
 				continue
 			}
@@ -703,7 +723,6 @@ func (cdc *Codec) decodeReflectBinaryStruct(bz []byte, info *TypeInfo, rv reflec
 		var lastFieldNum uint32
 		// Read each field.
 		for _, field := range info.Fields {
-
 			// Get field rv and info.
 			var frv = rv.Field(field.Index)
 			var finfo *TypeInfo
@@ -714,7 +733,7 @@ func (cdc *Codec) decodeReflectBinaryStruct(bz []byte, info *TypeInfo, rv reflec
 
 			// We're done if we've consumed all the bytes.
 			if len(bz) == 0 {
-				frv.Set(reflect.Zero(frv.Type()))
+				frv.Set(defaultValue(frv.Type()))
 				continue
 			}
 
@@ -731,7 +750,7 @@ func (cdc *Codec) decodeReflectBinaryStruct(bz []byte, info *TypeInfo, rv reflec
 				fnum, typ, _n, err = decodeFieldNumberAndTyp3(bz)
 				if field.BinFieldNum < fnum {
 					// Set zero field value.
-					frv.Set(reflect.Zero(frv.Type()))
+					frv.Set(defaultValue(frv.Type()))
 					continue
 					// Do not slide, we will read it again.
 				}
@@ -760,7 +779,6 @@ func (cdc *Codec) decodeReflectBinaryStruct(bz []byte, info *TypeInfo, rv reflec
 						typWanted, fnum, info.Type, typ))
 					return
 				}
-
 				// Decode field into frv.
 				_n, err = cdc.decodeReflectBinary(bz, finfo, frv, field.FieldOptions, false)
 				if slide(&bz, &n, _n) && err != nil {

binary-decode.go

@@ -24,10 +24,10 @@ import (
 // CONTRACT: rv is valid.
 func (cdc *Codec) encodeReflectBinary(w io.Writer, info *TypeInfo, rv reflect.Value, fopts FieldOptions, bare bool) (err error) {
 	if rv.Kind() == reflect.Ptr {
-		panic("should not happen")
+		panic("not allowed to be called with a reflect.Ptr")
 	}
 	if !rv.IsValid() {
-		panic("should not happen")
+		panic("not allowed to be called with invalid / zero Value")
 	}
 	if printLog {
 		spew.Printf("(E) encodeReflectBinary(info: %v, rv: %#v (%v), fopts: %v)\n",
@@ -297,6 +297,9 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec
 			}
 		}
 	} else {
+		// NOTE: ert is for the element value, while einfo.Type is dereferenced.
+		isErtStructPointer := ert.Kind() == reflect.Ptr && einfo.Type.Kind() == reflect.Struct
+
 		// Write elems in unpacked form.
 		for i := 0; i < rv.Len(); i++ {
 			// Write elements as repeated fields of the parent struct.
@@ -307,6 +310,17 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec
 			// Get dereferenced element value and info.
 			var erv, isDefault = isDefaultValue(rv.Index(i))
 			if isDefault {
+				// Special case if:
+				//  - erv is a struct pointer and
+				//  - field option has EmptyElements set
+				if isErtStructPointer && fopts.EmptyElements {
+					// NOTE: Not sure what to do here, but for future-proofing,
+					// we explicitly fail on nil pointers, just like
+					// Proto3's Golang client does.
+					// This also makes it easier to upgrade to Amino2
+					// which would enable the encoding of nil structs.
+					return errors.New("nil struct pointers not supported when empty_elements field tag is set")
+				}
 				// Nothing to encode, so the length is 0.
 				err = EncodeByte(buf, byte(0x00))
 				if err != nil {
@@ -384,28 +398,42 @@ func (cdc *Codec) encodeReflectBinaryStruct(w io.Writer, info *TypeInfo, rv refl
 				return
 			}
 			// Get dereferenced field value and info.
-			var frv, isDefault = isDefaultValue(rv.Field(field.Index))
-			if isDefault {
-				// Do not encode default value fields.
+			var frv = rv.Field(field.Index)
+			var frvIsPtr = frv.Kind() == reflect.Ptr
+			var dfrv, isDefault = isDefaultValue(frv)
+			if isDefault && !fopts.WriteEmpty {
+				// Do not encode default value fields
+				// (except when `amino:"write_empty"` is set).
 				continue
 			}
 			if field.UnpackedList {
 				// Write repeated field entries for each list item.
-				err = cdc.encodeReflectBinaryList(buf, finfo, frv, field.FieldOptions, true)
+				err = cdc.encodeReflectBinaryList(buf, finfo, dfrv, field.FieldOptions, true)
 				if err != nil {
 					return
 				}
 			} else {
+				lBeforeKey := buf.Len()
 				// Write field key (number and type).
 				err = encodeFieldNumberAndTyp3(buf, field.BinFieldNum, typeToTyp3(finfo.Type, field.FieldOptions))
 				if err != nil {
 					return
 				}
-				// Write field from rv.
-				err = cdc.encodeReflectBinary(buf, finfo, frv, field.FieldOptions, false)
+				lBeforeValue := buf.Len()
+
+				// Write field value from rv.
+				err = cdc.encodeReflectBinary(buf, finfo, dfrv, field.FieldOptions, false)
 				if err != nil {
 					return
 				}
+				lAfterValue := buf.Len()
+
+				if !frvIsPtr && !fopts.WriteEmpty && lBeforeValue == lAfterValue-1 && buf.Bytes()[buf.Len()-1] == 0x00 {
+					// rollback typ3/fieldnum and last byte if
+					// not a pointer and empty:
+					buf.Truncate(lBeforeKey)
+				}
+
 			}
 		}
 	}