Foreach hash id arg (#3716)

* add the option to hash the id when a string value is used

* add debug log to display the mapping between string keys and hash values

* changelog update

Author: wildum

CHANGELOG.md

@@ -14,6 +14,10 @@ Main (unreleased)
 
 - (_Experimental_) Add an `array.group_by` stdlib function to group items in an array by a key. (@wildum)
 
+### Enhancements
+
+- Add `hash_string_id` argument to `foreach` block to hash the string representation of the pipeline id instead of using the string itself. (@wildum)
+
 v1.9.0-rc.0
 -----------------
 

docs/sources/reference/config-blocks/foreach.md

@@ -35,9 +35,12 @@ You can use the following arguments with `foreach`:
 | `var`            | `string`    | Name of the variable referring to the current item in the collection.                    |         | yes      |
 | `enable_metrics` | `bool`      | Whether to expose debug metrics in the {{< param "PRODUCT_NAME" >}} `/metrics` endpoint. | `false` | no       |
 | `id`             | `string`    | Name of the field to use from collection items for child component's identification.     | `""`    | no       |
+| `hash_string_id` | `bool`      | Whether to hash the string representation of the id of the collection items.             | `false` | no       |
 
 The items in the `collection` list can be of any type [type][types], such as a bool, a string, a list, or a map.
 
+When using a collection of strings or when the `id` field is a string, you can set `hash_string_id` to `true` to hash the string representation of the `id` field instead of using the string itself. This is recommended when the strings are long because the values will be used to identify the components that are created dynamically in metrics, logs and in the UI.
+
 {{< admonition type="warning" >}}
 Setting `enable_metrics` to `true` when `collection` has lots of elements may cause a large number of metrics to appear on the {{< param "PRODUCT_NAME" >}} `/metric` endpoint.
 {{< /admonition >}}

internal/nodeconf/foreach/foreach.go

@@ -12,9 +12,10 @@ const (
 )
 
 type Arguments struct {
-	Collection []any  `alloy:"collection,attr"`
-	Var        string `alloy:"var,attr"`
-	Id         string `alloy:"id,attr,optional"`
+	Collection   []any  `alloy:"collection,attr"`
+	Var          string `alloy:"var,attr"`
+	Id           string `alloy:"id,attr,optional"`
+	HashStringId bool   `alloy:"hash_string_id,attr,optional"`
 
 	// EnableMetrics should be false by default.
 	// That way users are protected from an explosion of debug metrics

internal/runtime/internal/controller/node_config_foreach.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"hash/fnv"
 	"path"
+	"reflect"
 	"strings"
 	"sync"
 	"time"
@@ -212,16 +213,20 @@ func (fn *ForeachConfigNode) evaluate(scope *vm.Scope) error {
 		// We must create an ID from the collection entries to avoid recreating all components on every updates.
 		// We track the hash counts because the collection might contain duplicates ([1, 1, 1] would result in the same ids
 		// so we handle it by adding the count at the end -> [11, 12, 13]
-		customComponentID := fmt.Sprintf("foreach_%s", objectFingerprint(id))
+		customComponentID := fmt.Sprintf("foreach_%s", objectFingerprint(id, args.HashStringId))
 		count := fn.customComponentHashCounts[customComponentID] // count = 0 if the key is not found
 		fn.customComponentHashCounts[customComponentID] = count + 1
 		customComponentID += fmt.Sprintf("_%d", count+1)
 
-		cc, err := fn.getOrCreateCustomComponent(customComponentID)
+		cc, created, err := fn.getOrCreateCustomComponent(customComponentID)
 		if err != nil {
 			return err
 		}
 
+		if created && args.HashStringId && id != nil && reflect.TypeOf(id).Kind() == reflect.String {
+			level.Debug(fn.logger).Log("msg", "a new foreach pipeline was created", "value", id, "fingerprint", customComponentID)
+		}
+
 		// Expose the current scope + the collection item that correspond to the child.
 		vars := deepCopyMap(scope.Variables)
 		vars[args.Var] = args.Collection[i]
@@ -253,18 +258,18 @@ func (fn *ForeachConfigNode) evaluate(scope *vm.Scope) error {
 
 // Assumes that a lock is held,
 // so that fn.moduleController doesn't change while the function is running.
-func (fn *ForeachConfigNode) getOrCreateCustomComponent(customComponentID string) (CustomComponent, error) {
+func (fn *ForeachConfigNode) getOrCreateCustomComponent(customComponentID string) (CustomComponent, bool, error) {
 	cc, exists := fn.customComponents[customComponentID]
 	if exists {
-		return cc, nil
+		return cc, false, nil
 	}
 
 	newCC, err := fn.moduleController.NewCustomComponent(customComponentID, func(exports map[string]any) {})
 	if err != nil {
-		return nil, fmt.Errorf("creating custom component: %w", err)
+		return nil, true, fmt.Errorf("creating custom component: %w", err)
 	}
 	fn.customComponents[customComponentID] = newCC
-	return newCC, nil
+	return newCC, true, nil
 }
 
 func (fn *ForeachConfigNode) UpdateBlock(b *ast.BlockStmt) {
@@ -420,10 +425,13 @@ func computeHash(s string) string {
 	return hex.EncodeToString(hasher.Sum(nil))
 }
 
-func objectFingerprint(id any) string {
+func objectFingerprint(id any, hashId bool) string {
 	// TODO: Test what happens if there is a "true" string and a true bool in the collection.
 	switch v := id.(type) {
 	case string:
+		if hashId {
+			return computeHash(v)
+		}
 		return replaceNonAlphaNumeric(v)
 	case int, bool:
 		return fmt.Sprintf("%v", v)

internal/runtime/internal/controller/node_config_foreach_test.go

@@ -276,6 +276,68 @@ func TestNonAlphaNumericString3(t *testing.T) {
 	require.ElementsMatch(t, customComponentIds, []string{"foreach_123__s4_1", "foreach_123__s4_1_1"})
 }
 
+func TestStringIDHash(t *testing.T) {
+	config := `foreach "default" {
+		collection = ["123./st%4$"]
+		var = "num"
+		hash_string_id = true
+		template {
+		}
+	}`
+	foreachConfigNode := NewForeachConfigNode(getBlockFromConfig(t, config), getComponentGlobals(t), nil)
+	require.NoError(t, foreachConfigNode.Evaluate(vm.NewScope(make(map[string]interface{}))))
+	customComponentIds := foreachConfigNode.moduleController.(*ModuleControllerMock).CustomComponents
+	require.ElementsMatch(t, customComponentIds, []string{"foreach_1951d330e1267d082c816bfb3f40cce6eb9a8da9f6a6b9da09ace3c6514361cd_1"})
+}
+
+func TestStringIDHashWithKey(t *testing.T) {
+	config := `foreach "default" {
+		collection = [obj1, obj2]
+		var = "num"
+		hash_string_id = true
+		id = "label1"
+		template {
+		}
+	}`
+	foreachConfigNode := NewForeachConfigNode(getBlockFromConfig(t, config), getComponentGlobals(t), nil)
+	vars := map[string]interface{}{
+		"obj1": map[string]string{
+			"label1": "123./st%4$",
+			"label2": "b",
+		},
+		"obj2": map[string]string{
+			"label1": "aaaaaaaaaaaaaaabbbbbbbbbcccccccccdddddddddeeeeeeeeeffffffffffgggggggggggghhhhhhhhhhiiiiiiiiiiijjjjjjjjjjjkkkkkkkkkklllll",
+		},
+	}
+	require.NoError(t, foreachConfigNode.Evaluate(vm.NewScope(vars)))
+	customComponentIds := foreachConfigNode.moduleController.(*ModuleControllerMock).CustomComponents
+	require.ElementsMatch(t, customComponentIds, []string{"foreach_1951d330e1267d082c816bfb3f40cce6eb9a8da9f6a6b9da09ace3c6514361cd_1", "foreach_986cb398ec7d2d70a69bab597e8525ccc5c67594765a82ee7d0f011937cdec25_1"})
+}
+
+func TestStringIDHashWithKeySameValue(t *testing.T) {
+	config := `foreach "default" {
+		collection = [obj1, obj2]
+		var = "num"
+		hash_string_id = true
+		id = "label1"
+		template {
+		}
+	}`
+	foreachConfigNode := NewForeachConfigNode(getBlockFromConfig(t, config), getComponentGlobals(t), nil)
+	vars := map[string]interface{}{
+		"obj1": map[string]string{
+			"label1": "123./st%4$",
+			"label2": "b",
+		},
+		"obj2": map[string]string{
+			"label1": "123./st%4$",
+		},
+	}
+	require.NoError(t, foreachConfigNode.Evaluate(vm.NewScope(vars)))
+	customComponentIds := foreachConfigNode.moduleController.(*ModuleControllerMock).CustomComponents
+	require.ElementsMatch(t, customComponentIds, []string{"foreach_1951d330e1267d082c816bfb3f40cce6eb9a8da9f6a6b9da09ace3c6514361cd_1", "foreach_1951d330e1267d082c816bfb3f40cce6eb9a8da9f6a6b9da09ace3c6514361cd_2"})
+}
+
 func TestCollectionNonArrayValue(t *testing.T) {
 	config := `foreach "default" {
 		collection = "aaa"