Refactor date period last event now (#116)

* call m.Run() in TestMain

* TDD refactor date period and bindTime

* refactor date period and bindTime

* update docs

* fix typo in TestMain

* better docs on period

* extract nearTime logic

* extract generation logic

Author: Andrea Spacca

docs/fields-configuration.md

@@ -13,7 +13,7 @@ For each config entry the following fields are available:
 - `fuzziness` *optional (`long` and `double` type only)*: when generating data you could want generated values to change in a known interval. Fuzziness allow to specify the maximum delta a generated value can have from the previous value (for the same field), as a delta percentage; value must be between 0.0 and 1.0, where 0 is 0% and 1 is 100%. When not specified there is no constraint on the generated values, boundaries will be defined by the underlying field type
 - `range` *optional (`long` and `double` type only)*: value will be generated between `min` and `max`
 - `cardinality` *optional*: number of different values for the field; note that this value may not be respected if not enough events are generated. Es `cardinality: 1000` with `100` generated events would produce `100` different values, not `1000`.
-- `period` *optional (`date` type only)*: values will be evenly generated between `time.Now()` and `time.Now().Add(period)`, where period is expressed as `time.Duration`
+- `period` *optional (`date` type only)*: values will be evenly generated between `time.Now()` and `time.Now().Add(period)`, where period is expressed as `time.Duration`. It accepts also a negative duration: in this case  values will be evenly generated between `time.Now().Add(period)` and `time.Now()`.
 - `object_keys` *optional (`object` type only)*: list of field names to generate in a object field type; if not specified a random number of field names will be generated in the object filed type
 - `value` *optional*: hardcoded value to set for the field (any `cardinality` will be ignored)
 - `enum` *optional (`keyword` type only)*: list of strings to randomly chose from a value to set for the field (any `cardinality` will be applied limited to the size of the `enum` values)

docs/performances.md

@@ -56,7 +56,7 @@ _GeneratorTextTemplateVPCFlowLogs-16      95.0 ± 0%
 
 ```
 
-If you are curios how those benchmarks translate to time needed for generating dataset, we ran some test runs monitoring the execution times.
+If you are curious how those benchmarks translate to time needed for generating dataset, we ran some test runs monitoring the execution times.
 We generated directly from the built binaries 20GB of "aws dynamodb 1.28.3" Schema C data.
 
 ```

pkg/genlib/generator_interface.go

@@ -48,8 +48,8 @@ const (
 	FieldTypeFlattened       = "flattened"
 	FieldTypeGeoPoint        = "geo_point"
 
-	FieldTypeTimeRange  = 3600 // seconds
-	FieldTypeTimeLayout = "2006-01-02T15:04:05.999999Z07:00"
+	FieldTypeDurationSpan = 1000 // milliseconds
+	FieldTypeTimeLayout   = "2006-01-02T15:04:05.999999Z07:00"
 )
 
 var (
@@ -338,7 +338,7 @@ func genNounsNWithReturn(n int) string {
 	return value
 }
 
-func randGeoPoint(buf *bytes.Buffer) error {
+func randGeoPoint() (int, int, int, int) {
 	lat := customRand.Intn(181) - 90
 	var latD int
 	if lat != -90 && lat != 90 {
@@ -349,23 +349,8 @@ func randGeoPoint(buf *bytes.Buffer) error {
 	if long != -180 && long != 180 {
 		longD = customRand.Intn(100)
 	}
-	_, err := fmt.Fprintf(buf, "%d.%d,%d.%d", lat, latD, long, longD)
-	return err
-}
 
-func randGeoPointWithReturn() string {
-	lat := customRand.Intn(181) - 90
-	var latD int
-	if lat != -90 && lat != 90 {
-		latD = customRand.Intn(100)
-	}
-	var longD int
-	long := customRand.Intn(361) - 180
-	if long != -180 && long != 180 {
-		longD = customRand.Intn(100)
-	}
-
-	return fmt.Sprintf("%d.%d,%d.%d", lat, latD, long, longD)
+	return lat, latD, long, longD
 }
 
 func bindConstantKeyword(field Field, fieldMap map[string]any) error {
@@ -396,19 +381,7 @@ func bindKeyword(fieldCfg ConfigField, field Field, fieldMap map[string]any) err
 
 		fieldMap[field.Name] = emitFNotReturn
 	} else if len(field.Example) > 0 {
-
-		totWords := len(keywordRegex.Split(field.Example, -1))
-
-		var joiner string
-		if strings.Contains(field.Example, "\\.") {
-			joiner = "\\."
-		} else if strings.Contains(field.Example, "-") {
-			joiner = "-"
-		} else if strings.Contains(field.Example, "_") {
-			joiner = "_"
-		} else if strings.Contains(field.Example, " ") {
-			joiner = " "
-		}
+		totWords, joiner := totWordsAndJoiner(field.Example)
 
 		return bindJoinRand(field, totWords, joiner, fieldMap)
 	} else {
@@ -424,6 +397,22 @@ func bindKeyword(fieldCfg ConfigField, field Field, fieldMap map[string]any) err
 	return nil
 }
 
+func totWordsAndJoiner(fieldExample string) (int, string) {
+	totWords := len(keywordRegex.Split(fieldExample, -1))
+
+	var joiner string
+	if strings.Contains(fieldExample, "\\.") {
+		joiner = "\\."
+	} else if strings.Contains(fieldExample, "-") {
+		joiner = "-"
+	} else if strings.Contains(fieldExample, "_") {
+		joiner = "_"
+	} else if strings.Contains(fieldExample, " ") {
+		joiner = " "
+	}
+
+	return totWords, joiner
+}
 func bindJoinRand(field Field, N int, joiner string, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *genState, buf *bytes.Buffer) error {
@@ -477,7 +466,9 @@ func bindBool(field Field, fieldMap map[string]any) error {
 func bindGeoPoint(field Field, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *genState, buf *bytes.Buffer) error {
-		return randGeoPoint(buf)
+		lat, latD, long, longD := randGeoPoint()
+		_, err := fmt.Fprintf(buf, "%d.%d,%d.%d", lat, latD, long, longD)
+		return err
 	}
 
 	fieldMap[field.Name] = emitFNotReturn
@@ -498,14 +489,7 @@ func bindWordN(field Field, n int, fieldMap map[string]any) error {
 func bindNearTime(fieldCfg ConfigField, field Field, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *genState, buf *bytes.Buffer) error {
-		var offset time.Duration
-		if fieldCfg.Period > 0 && state.totEvents > 0 {
-			offset = time.Duration((fieldCfg.Period.Nanoseconds() / int64(state.totEvents)) * int64(state.counter))
-		} else {
-			offset = time.Duration(customRand.Intn(FieldTypeTimeRange)*-1) * time.Second
-		}
-
-		newTime := timeNowToBind.Add(offset)
+		newTime := nearTime(fieldCfg, state)
 
 		buf.WriteString(newTime.Format(FieldTypeTimeLayout))
 		return nil
@@ -514,13 +498,29 @@ func bindNearTime(fieldCfg ConfigField, field Field, fieldMap map[string]any) er
 	return nil
 }
 
+func nearTime(fieldCfg ConfigField, state *genState) time.Time {
+	var offset time.Duration
+	if fieldCfg.Period > 0 && state.totEvents > 0 {
+		offset = time.Duration((fieldCfg.Period.Nanoseconds() / int64(state.totEvents)) * int64(state.counter))
+	} else if fieldCfg.Period < 0 && state.totEvents > 0 {
+		offset = time.Duration((fieldCfg.Period.Nanoseconds() / int64(state.totEvents)) * (int64(state.totEvents - state.counter)))
+	} else {
+		offset = time.Duration(customRand.Intn(FieldTypeDurationSpan)) * time.Millisecond
+	}
+
+	newTime := timeNowToBind.Add(offset)
+
+	if state.totEvents <= 0 {
+		timeNowToBind = newTime
+	}
+
+	return newTime
+}
+
 func bindIP(field Field, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *genState, buf *bytes.Buffer) error {
-		i0 := customRand.Intn(255)
-		i1 := customRand.Intn(255)
-		i2 := customRand.Intn(255)
-		i3 := customRand.Intn(255)
+		i0, i1, i2, i3 := randIP()
 
 		_, err := fmt.Fprintf(buf, "%d.%d.%d.%d", i0, i1, i2, i3)
 		return err
@@ -744,19 +744,7 @@ func bindKeywordWithReturn(fieldCfg ConfigField, field Field, fieldMap map[strin
 
 		fieldMap[field.Name] = emitF
 	} else if len(field.Example) > 0 {
-
-		totWords := len(keywordRegex.Split(field.Example, -1))
-
-		var joiner string
-		if strings.Contains(field.Example, "\\.") {
-			joiner = "\\."
-		} else if strings.Contains(field.Example, "-") {
-			joiner = "-"
-		} else if strings.Contains(field.Example, "_") {
-			joiner = "_"
-		} else if strings.Contains(field.Example, " ") {
-			joiner = " "
-		}
+		totWords, joiner := totWordsAndJoiner(field.Example)
 
 		return bindJoinRandWithReturn(field, totWords, joiner, fieldMap)
 	} else {
@@ -818,7 +806,8 @@ func bindBoolWithReturn(field Field, fieldMap map[string]any) error {
 func bindGeoPointWithReturn(field Field, fieldMap map[string]any) error {
 	var emitF EmitF
 	emitF = func(state *genState) any {
-		return randGeoPointWithReturn()
+		lat, latD, long, longD := randGeoPoint()
+		return fmt.Sprintf("%d.%d,%d.%d", lat, latD, long, longD)
 	}
 
 	fieldMap[field.Name] = emitF
@@ -838,36 +827,32 @@ func bindWordNWithReturn(field Field, n int, fieldMap map[string]any) error {
 func bindNearTimeWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]any) error {
 	var emitF EmitF
 	emitF = func(state *genState) any {
-		var offset time.Duration
-		if fieldCfg.Period > 0 {
-			offset = time.Duration((fieldCfg.Period.Nanoseconds() / int64(state.totEvents)) * int64(state.counter))
-		} else {
-			offset = time.Duration(customRand.Intn(FieldTypeTimeRange)*-1) * time.Second
-		}
-
-		newTime := timeNowToBind.Add(offset)
-
-		return newTime
+		return nearTime(fieldCfg, state)
 	}
+
 	fieldMap[field.Name] = emitF
 	return nil
 }
 
 func bindIPWithReturn(field Field, fieldMap map[string]any) error {
 	var emitF EmitF
 	emitF = func(state *genState) any {
-		i0 := customRand.Intn(255)
-		i1 := customRand.Intn(255)
-		i2 := customRand.Intn(255)
-		i3 := customRand.Intn(255)
+		i0, i1, i2, i3 := randIP()
 
 		return fmt.Sprintf("%d.%d.%d.%d", i0, i1, i2, i3)
 	}
 
 	fieldMap[field.Name] = emitF
 	return nil
 }
+func randIP() (int, int, int, int) {
+	i0 := customRand.Intn(255)
+	i1 := customRand.Intn(255)
+	i2 := customRand.Intn(255)
+	i3 := customRand.Intn(255)
 
+	return i0, i1, i2, i3
+}
 func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]any) error {
 	dummyFunc := makeIntFunc(fieldCfg, field)
 

pkg/genlib/generator_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/fields"
 	"log"
 	"math/rand"
+	"os"
 	"testing"
 	"time"
 )
@@ -21,6 +22,7 @@ func TestMain(m *testing.M) {
 	InitGeneratorRandSeed(randSeed)
 	InitGeneratorTimeNow(timeNow)
 
+	os.Exit(m.Run())
 }
 
 func Benchmark_GeneratorCustomTemplateJSONContent(b *testing.B) {

pkg/genlib/generator_with_custom_template_test.go

@@ -442,30 +442,25 @@ func Test_FieldDateWithCustomTemplate(t *testing.T) {
 	t.Logf("with template: %s", string(template))
 	nSpins := rand.Intn(1024) + 1
 	for i := 0; i < nSpins; i++ {
-		now := time.Now()
+		previous := timeNowToBind
 
 		b := testSingleTWithCustomTemplate[string](t, fld, nil, template)
 
 		if ts, err := time.Parse(FieldTypeTimeLayout, b); err != nil {
 			t.Errorf("Fail parse timestamp %v", err)
 		} else {
-			// Timestamp should be +- FieldTypeDurationSpan from now within a second of slop
-			ts.Add(time.Second * -1)
-			ts.Add(time.Second)
-
-			diff := ts.Sub(now)
-			if diff < 0 {
-				diff = -diff
+			// Timestamp should be from now within a FieldTypeDurationSpan milliseconds of slop
+			diff := ts.Sub(previous)
+			if diff < 0 || diff > FieldTypeDurationSpan*time.Millisecond {
+				t.Errorf("Data generated before now, diff: %v", diff)
 			}
 
-			if diff >= FieldTypeTimeRange*time.Second {
-				t.Errorf("Date generated out of span range %v", diff)
-			}
+			previous = ts
 		}
 	}
 }
 
-func Test_FieldDateAndPeriodWithCustomTemplate(t *testing.T) {
+func Test_FieldDateAndPeriodPositiveWithCustomTemplate(t *testing.T) {
 	fld := Field{
 		Name: "alpha",
 		Type: FieldTypeDate,
@@ -517,6 +512,58 @@ func Test_FieldDateAndPeriodWithCustomTemplate(t *testing.T) {
 	}
 }
 
+func Test_FieldDateAndPeriodNegativeWithCustomTemplate(t *testing.T) {
+	fld := Field{
+		Name: "alpha",
+		Type: FieldTypeDate,
+	}
+
+	template := []byte(`{"alpha":"{{.alpha}}"}`)
+	configYaml := []byte("fields:\n  - name: alpha\n    period: -10s")
+	t.Logf("with template: %s", string(template))
+
+	cfg, err := config.LoadConfigFromYaml(configYaml)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, 10)
+
+	var buf bytes.Buffer
+
+	nSpins := 10
+	for i := 0; i < nSpins; i++ {
+		if err := g.Emit(&buf); err != nil {
+			t.Fatal(err)
+		}
+
+		m := unmarshalJSONT[string](t, buf.Bytes())
+		buf.Reset()
+
+		if len(m) != 1 {
+			t.Errorf("Expected map size 1, got %d", len(m))
+		}
+
+		v, ok := m[fld.Name]
+
+		if !ok {
+			t.Errorf("Missing key %v", fld.Name)
+		}
+
+		if ts, err := time.Parse(FieldTypeTimeLayout, v); err != nil {
+			t.Errorf("Fail parse timestamp %v", err)
+		} else {
+			// Timestamp should be +1s for every iteration
+			expectedTime := timeNowToBind.Truncate(time.Millisecond).Add(-10*time.Second + time.Second*time.Duration(i))
+
+			diff := expectedTime.Sub(ts.Truncate(time.Millisecond))
+			if diff != 0 {
+				t.Errorf("Date generated out of period range %v", diff)
+			}
+		}
+	}
+}
+
 func Test_FieldIPWithCustomTemplate(t *testing.T) {
 	fld := Field{
 		Name: "alpha",