[Osquerybeat] [Bugfix] Encoding timestamps

changelog/fragments/1764715501-encoding_timestamps.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: fixes zero time encoding for unix timestamps
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component:
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding.go

@@ -313,78 +313,87 @@ func convertValueToStringWithTag(fieldValue reflect.Value, flag EncodingFlag, ta
 	}
 }
 
+var timeLayouts = map[string]string{
+	"rfc3339":     time.RFC3339,
+	"rfc3339nano": time.RFC3339Nano,
+	"rfc822":      time.RFC822,
+	"rfc822z":     time.RFC822Z,
+	"rfc850":      time.RFC850,
+	"rfc1123":     time.RFC1123,
+	"rfc1123z":    time.RFC1123Z,
+	"kitchen":     time.Kitchen,
+	"stamp":       time.Stamp,
+	"stampmilli":  time.StampMilli,
+	"stampmicro":  time.StampMicro,
+	"stampnano":   time.StampNano,
+}
+
 // formatTimeWithTagFormat formats a time.Time value with the specified format
 // and timezone conversion if specified in the tag.
 func formatTimeWithTagFormat(fieldValue reflect.Value, flag EncodingFlag, tag *reflect.StructTag) (string, error) {
-	// Check if the value is zero and the flag is not set to use numbers zero values
-	if !flag.has(EncodingFlagUseNumbersZeroValues) && fieldValue.IsZero() {
-		return "", nil
-	}
-
+	// Get the time.Time value from the field value
 	t, ok := fieldValue.Interface().(time.Time)
 	if !ok {
 		return "", fmt.Errorf("expected time.Time value but got %v", fieldValue.Type())
 	}
 
-	// If no tag is specified, use the default format
+	// If time is zero and the flag is not set to use numbers zero values, return empty string
+	if t.IsZero() && !flag.has(EncodingFlagUseNumbersZeroValues) {
+		return "", nil
+	}
+
+	// If no tag is specified, format the time using the default format
 	if tag == nil {
 		return t.Format(DefaultTimeFormat), nil
 	}
 
-	// Handle timezone conversion if specified in tag
+	// Get the timezone from the tag
+	timezone := DefaultTimezone
 	if tz, ok := tag.Lookup("tz"); ok {
-		loc, err := time.LoadLocation(tz)
-		if err != nil {
-			return "", fmt.Errorf("invalid timezone %s: %w", tz, err)
-		}
-		t = t.In(loc)
-	} else {
-		loc, err := time.LoadLocation(DefaultTimezone)
-		if err != nil {
-			return "", fmt.Errorf("invalid default timezone %s: %w", DefaultTimezone, err)
-		}
-		t = t.In(loc)
+		timezone = tz
 	}
+	loc, err := time.LoadLocation(timezone)
+	if err != nil {
+		return "", fmt.Errorf("invalid timezone %s: %w", timezone, err)
+	}
+	t = t.In(loc)
+
+	// if the format tag is specified, format the time using the specified format
+	if formatTag, ok := tag.Lookup("format"); ok {
+
+		// lower the format tag to make it case insensitive
+		format := strings.ToLower(formatTag)
+
+		// unix time formats require special handling, they are not a time layout
+		// and must be handled with a string conversion.
+		if strings.HasPrefix(format, "unix") {
+			// If the time is zero, return "0" for all unix time formats.
+			// Because osquery (sqlite) expects a zero unix timestamp to be represented as "0".
+			// If we don't handle this case, go will return the unix equivalent of go zero time (0001-01-01 00:00:00 +0000 UTC),
+			// which would result in a negative unix timestamp.
+			if t.IsZero() {
+				return "0", nil
+			}
+			switch format {
+			case "unix":
+				return strconv.FormatInt(t.Unix(), 10), nil
+			case "unixnano":
+				return strconv.FormatInt(t.UnixNano(), 10), nil
+			case "unixmilli":
+				return strconv.FormatInt(t.UnixMilli(), 10), nil
+			case "unixmicro":
+				return strconv.FormatInt(t.UnixMicro(), 10), nil
+			default:
+				return "", fmt.Errorf("unsupported unix time format: %s", format)
+			}
+		}
 
-	var result string
-	if timeFormat, ok := tag.Lookup("format"); ok {
-		switch strings.ToLower(timeFormat) {
-		case "unix":
-			result = strconv.FormatInt(t.Unix(), 10)
-		case "unixnano":
-			result = strconv.FormatInt(t.UnixNano(), 10)
-		case "unixmilli":
-			result = strconv.FormatInt(t.UnixMilli(), 10)
-		case "unixmicro":
-			result = strconv.FormatInt(t.UnixMicro(), 10)
-		case "rfc3339":
-			result = t.Format(time.RFC3339)
-		case "rfc3339nano":
-			result = t.Format(time.RFC3339Nano)
-		case "rfc822":
-			result = t.Format(time.RFC822)
-		case "rfc822z":
-			result = t.Format(time.RFC822Z)
-		case "rfc850":
-			result = t.Format(time.RFC850)
-		case "rfc1123":
-			result = t.Format(time.RFC1123)
-		case "rfc1123z":
-			result = t.Format(time.RFC1123Z)
-		case "kitchen":
-			result = t.Format(time.Stamp)
-		case "stampmilli":
-			result = t.Format(time.StampMilli)
-		case "stampmicro":
-			result = t.Format(time.StampMicro)
-		case "stampnano":
-			result = t.Format(time.StampNano)
-		default:
-			return "", fmt.Errorf("unsupported time format: %s", timeFormat)
+		if layout, ok := timeLayouts[format]; ok {
+			return t.Format(layout), nil
 		}
-	} else {
-		result = t.Format(time.RFC3339)
+		return "", fmt.Errorf("unsupported time format: %s", format)
 	}
 
-	return result, nil
+	// If no format is specified, format the time using the default format
+	return t.Format(DefaultTimeFormat), nil
 }

x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go

@@ -337,7 +337,7 @@
 			fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)),
 			flag:       0,
 			tag:        tagPtr(`format:"kitchen"`),
-			want:       "Jun 15 14:30:00",
+			want:       "2:30PM",
 			wantErr:    false,
 		},
 		{
@@ -404,6 +404,14 @@
 			want:       "0001-01-01T00:00:00Z",
 			wantErr:    false,
 		},
+		{
+			name:       "Zero unix timestamp with UseNumbersZeroValues flag",
+			fieldValue: reflect.ValueOf(time.Time{}),
+			flag:       EncodingFlagUseNumbersZeroValues,
+			tag:        tagPtr(`format:"unix"`),
+			want:       "0",
+			wantErr:    false,
+		},
 		{
 			name:       "With timezone Asia/Tokyo",
 			fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)),
@@ -799,7 +807,7 @@
 func TestGenerateColumnDefinitions_unexportedFields(t *testing.T) {
 	type testStruct struct {
 		Public    string `osquery:"public"`
 		private   string `osquery:"private"` //nolint:unused -- meaningful for test coverage
 		Exported2 int    `osquery:"exported2"`
 	}
 	expected := []table.ColumnDefinition{