I try to use the grule in product env, but got "fatal error: concurrent map read and map write" #477
Description
Here it's my codes as below.
I try to wrap the grule into a function which can be used by services' codes. As I know, if the rule has been built once, it should be cached and I can use NewKnowledgeBaseInstance function to get a copy of the knowledge base instance.
But when I use the codes as below in the product env, it returns "fatal error: concurrent map read and map write". The full stack is under the codes.
Is there anything wrong with my codes? Or Cann't I use the parsing cache feature in the product env?
BTW, the go and lib version:
go mod 1.19
github.com/hyperjumptech/grule-rule-engine v1.15.0
ExecuteGrule function
package grule
import (
"code.byted.org/gopkg/logs"
"code.byted.org/oec/oec_txg_common/utils"
"context"
"fmt"
"github.com/hyperjumptech/grule-rule-engine/ast"
"github.com/hyperjumptech/grule-rule-engine/builder"
"github.com/hyperjumptech/grule-rule-engine/engine"
"github.com/hyperjumptech/grule-rule-engine/pkg"
"github.com/pkg/errors"
"github.com/samber/lo"
"math/rand"
"reflect"
"sync"
"time"
)
const (
GruleExtVarName = "ext"
GruleRetVarName = "ret"
)
type GruleParam struct {
RuleName string `json:"name"` // Grule rule name
RuleVersion string `json:"version"` // Grule rule version
Rule string `json:"rule"` // Grule rule
FactorMap map[string]any `json:"factor_map,omitempty"` // Factor map: key is factor name, value is factor value
VariableMap map[string]any `json:"variable_map,omitempty"` // Variable map: key is variable name, value is variable value
}
func (s *GruleParam) String() string {
return fmt.Sprintf("[name: %s, version: %s, rule: %s, factor: %v, variable: %v]",
s.RuleName, s.RuleVersion, s.Rule, s.FactorMap, s.VariableMap)
}
type GruleRet struct {
IsMatched bool `json:"is_matched"`
}
func (s *GruleRet) String() string {
return fmt.Sprintf("[is_matched: %v]", s.IsMatched)
}
var (
knowledgeLib = ast.NewKnowledgeLibrary()
ruleBuilder = builder.NewRuleBuilder(knowledgeLib)
gruleEngine = engine.NewGruleEngine()
builtRules sync.Map // For recording built rules
buildRuleMutex sync.Mutex // For avoiding building rules concurrently
)
// ExecuteGrule
// Execute grule based on factor map, then return the result and error
func ExecuteGrule(ctx context.Context, param *GruleParam) (ret *GruleRet, err error) {
if param == nil || len(param.RuleName) == 0 || len(param.RuleVersion) == 0 ||
len(param.Rule) == 0 || param.FactorMap == nil {
return nil, errors.Errorf("param must be valid, param: %v", param)
}
start := time.Now()
defer func() {
if p := recover(); p != nil {
err = lo.Ternary(err == nil, errors.Errorf("panic: %+v", p),
errors.WithMessagef(err, "panic: %+v", p))
}
if err != nil {
logs.CtxWarn(ctx, "[ExecuteGrule] cost %v, param: %v, ret: %v, err: %+v",
time.Since(start), param, ret, err)
} else {
logs.CtxInfo(ctx, "[ExecuteGrule] cost %v, param: %v, ret: %v",
time.Since(start), param, ret)
}
}()
dataCtx := ast.NewDataContext()
for factor, value := range param.FactorMap {
// Add factors to the data context
if err := dataCtx.Add(factor, toValIfPtr(value)); err != nil {
return nil, errors.WithStack(err)
}
}
for variable, value := range param.VariableMap {
// Add variables to the data context
if err := dataCtx.Add(variable, toValIfPtr(value)); err != nil {
return nil, errors.WithStack(err)
}
}
// Add custom functions to the data context
if err := dataCtx.Add(GruleExtVarName, &GruleExt{}); err != nil {
return nil, errors.WithStack(err)
}
// Add ret to the data context
ret = &GruleRet{}
if err := dataCtx.Add(GruleRetVarName, ret); err != nil {
return nil, errors.WithStack(err)
}
// Build rule and get knowledge base
knowledgeBase, err := buildRule(ctx, param)
if err != nil {
return nil, errors.WithStack(err)
}
// Execute rule based on the data context
if err := gruleEngine.Execute(dataCtx, knowledgeBase); err != nil {
return nil, errors.WithStack(err)
}
return ret, nil
}
// Build rule and return knowledge base
func buildRule(ctx context.Context, param *GruleParam) (*ast.KnowledgeBase, error) {
if param == nil || len(param.RuleName) == 0 || len(param.RuleVersion) == 0 || len(param.Rule) == 0 {
return nil, errors.Errorf("param must be valid, param: %s", param)
}
// Check the rule is built or not before lock
if ruleVersion, ok := builtRules.Load(param.RuleName); ok && ruleVersion == param.RuleVersion {
// The rule is built, use it directly
return knowledgeLib.NewKnowledgeBaseInstance(param.RuleName, param.RuleVersion)
}
if !buildRuleMutex.TryLock() {
// Lock failed, use a random version to build the rule
nano := time.Now().UnixNano()
rand.Seed(nano)
randomVersion := fmt.Sprintf("random_version_%d_%d", nano, rand.Int())
if err := ruleBuilder.BuildRuleFromResource(
param.RuleName, randomVersion, pkg.NewBytesResource([]byte(param.Rule))); err != nil {
return nil, errors.WithStack(err)
}
logs.CtxInfo(ctx, "[buildRule] build rule with random version, "+
"ruleName: %s, ruleVersion: %s, randomVersion: %s", param.RuleName, param.RuleVersion, randomVersion)
return knowledgeLib.NewKnowledgeBaseInstance(param.RuleName, param.RuleVersion)
}
defer buildRuleMutex.Unlock()
// Lock successfully, double check
// Double-check the rule is built or not after locked
if ruleVersion, ok := builtRules.Load(param.RuleName); ok && ruleVersion == param.RuleVersion {
// Has been built by other goroutine, use it directly
return knowledgeLib.NewKnowledgeBaseInstance(param.RuleName, param.RuleVersion)
}
if err := ruleBuilder.BuildRuleFromResource(
param.RuleName, param.RuleVersion, pkg.NewBytesResource([]byte(param.Rule))); err != nil {
return nil, errors.WithStack(err)
}
builtRules.Store(param.RuleName, param.RuleVersion)
return knowledgeLib.NewKnowledgeBaseInstance(param.RuleName, param.RuleVersion)
}
// ToValIfPtr
// Convert val to value type if val is pointer type, otherwise return val itself.
func toValIfPtr(val any) any {
v := reflect.ValueOf(val)
if v.Kind() == reflect.Ptr && !v.IsNil() {
// if val is non-nil pointer, return its value
return v.Elem().Interface()
}
// return origin val if val isn't pointer
return val
}
// GruleExt
// grule-rule-engine
// https://github.com/hyperjumptech/grule-rule-engine/blob/master/docs/cn/Function_cn.md
type GruleExt struct {
}
// In
// Helper function to check if a value is in the target list.
// The target list should be non-empty.
// Example:
// - values: 1, targets: [1,2,3], return true
// - values: 4, targets: [1,2,3], return false
func (s *GruleExt) In(value any, targets ...any) bool {
targets = utils.FlattenSlice(targets, 2)
for _, target := range targets {
if utils.Equal(target, value) {
return true
}
}
return false
}
// Contains
// Helper function to check if the value list contains the target.
// The value list should be non-empty.
// Example:
// - values: [1,2,3], target: 1, return true
// - values: [1,2,3], target: 4, return false
func (s *GruleExt) Contains(values any, target any) bool {
anyList, ok := utils.ToAnySliceIfSlice(values)
if !ok {
return false
}
for _, value := range anyList.([]any) {
if utils.Equal(value, target) {
return true
}
}
return false
}
// ContainsAll
// Helper function to check if the value list contains all elements of the target list.
// The value list and target list should be non-empty.
// Example:
// - values: [1,2,3], targets: [1,2], return true
// - values: [1,2,3], targets: [2,4], return false
// - values: [1,2,3], targets: [4,5], return false
func (s *GruleExt) ContainsAll(values any, targets ...any) bool {
anyList, ok := utils.ToAnySliceIfSlice(values)
if !ok {
return false
}
targets = utils.FlattenSlice(targets, 2)
if len(anyList.([]any)) == 0 || len(targets) == 0 ||
len(anyList.([]any)) < len(targets) {
return false
}
// the logic below means the value list contains every target
return lo.EveryBy(targets, func(target any) bool {
return lo.ContainsBy(anyList.([]any), func(value any) bool {
return utils.Equal(value, target)
})
})
}
// NotContainsAny
// Helper function to check if the value list doesn't contain any elements of the target list
// The value list and target list should be non-empty.
// Example:
// - values: [1,2,3], targets: [1,2], return false
// - values: [1,2,3], targets: [2,4], return false
// - values: [1,2,3], targets: [4,5], return true
func (s *GruleExt) NotContainsAny(values any, targets ...any) bool {
anyList, ok := utils.ToAnySliceIfSlice(values)
if !ok {
return false
}
targets = utils.FlattenSlice(targets, 2)
if len(anyList.([]any)) == 0 || len(targets) == 0 {
return false
}
// the logic below means the value list doesn't contain any target
return lo.NoneBy(targets, func(target any) bool {
return lo.ContainsBy(anyList.([]any), func(value any) bool {
return utils.Equal(value, target)
})
})
}
// ContainsAny
// Helper function to check if the value list contains any elements of the target list
// The value list and target list should be non-empty.
// Example:
// - values: [1,2,3], targets: [1], return true
// - values: [1,2,3], targets: [1,4], return true
// - values: [1,2,3], targets: [4,5], return false
func (s *GruleExt) ContainsAny(values any, targets ...any) bool {
anyList, ok := utils.ToAnySliceIfSlice(values)
if !ok {
return false
}
targets = utils.FlattenSlice(targets, 2)
if len(anyList.([]any)) == 0 || len(targets) == 0 {
return false
}
// the logic below means the value list contains any of the target
return lo.SomeBy(targets, func(target any) bool {
return lo.ContainsBy(anyList.([]any), func(value any) bool {
return utils.Equal(value, target)
})
})
}fatal error: concurrent map read and map write
fatal error: concurrent map read and map write
goroutine 1511 [running]:
github.com/hyperjumptech/grule-rule-engine/ast.(*KnowledgeLibrary).NewKnowledgeBaseInstance(0xc01ac12bc8, {0xc01f22b200?, 0xc019693ad0?}, {0xc01f22b220, 0xd})
/opt/tiger/compile_path/pkg/mod/github.com/hyperjumptech/[email protected]/ast/KnowledgeBase.go:132 +0xe5
xxxxxxx/utils/grule.buildRule({0xd7cf948, 0xc01efc4ab0}, 0xc01bde22c0)
xxxxxxx/utils/grule/grule_exec.go:119 +0x185