diff --git a/contrib/drivers/mysql/mysql_z_unit_feature_with_all_four_layers_test.go b/contrib/drivers/mysql/mysql_z_unit_feature_with_all_four_layers_test.go new file mode 100644 index 00000000000..7e97dfc3d45 --- /dev/null +++ b/contrib/drivers/mysql/mysql_z_unit_feature_with_all_four_layers_test.go @@ -0,0 +1,314 @@ +// Copyright GoFrame Author(https://goframe.org). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package mysql_test + +import ( + "fmt" + "testing" + "time" + + "github.com/gogf/gf/v2/test/gtest" + "github.com/gogf/gf/v2/util/gmeta" +) + +// Test_WithAll_MediumDataset_FourLayers 中小型四层数据分批查询测试 +// 数据规模： +// - 20 个用户 +// - 每个用户 5 个 UserScore（共 100 个 score） +// - 每个 score 4 个 ScoreDetail（共 400 个 detail） +// - 每个 detail 3 个 DetailMeta（共 1200 个 meta） +// +// 测试目标： +// 1. 验证四层关联查询的正确性 +// 2. 验证每层都能正确触发分批查询 +// 3. 输出较短的 SQL 便于验证查询逻辑 +// 4. 对比默认查询和优化查询的差异 +func Test_WithAll_MediumDataset_FourLayers(t *testing.T) { + var ( + // 使用独立的表名避免与其他测试冲突 + tableUser = "user_four_layers" + tableUserDetail = "user_detail_four_layers" + tableUserScores = "user_scores_four_layers" + tableUserScoreDetails = "user_score_details_four_layers" + tableDetailMeta = "detail_meta_four_layers" + ) + + // 数据结构定义（四层） + type DetailMeta struct { + gmeta.Meta `orm:"table:detail_meta_four_layers"` + Id int `json:"id"` + DetailId int `json:"detail_id"` + MetaKey string `json:"meta_key"` + MetaValue string `json:"meta_value"` + } + + type UserScoreDetails struct { + gmeta.Meta `orm:"table:user_score_details_four_layers"` + Id int `json:"id"` + ScoreId int `json:"score_id"` + DetailInfo string `json:"detail_info"` + DetailMeta []*DetailMeta `orm:"with:detail_id=id,batch:threshold=100,batchSize=200"` + } + + type UserScores struct { + gmeta.Meta `orm:"table:user_scores_four_layers"` + Id int `json:"id"` + Uid int `json:"uid"` + Score int `json:"score"` + ScoreDetails []*UserScoreDetails `orm:"with:score_id=id"` + } + + type UserDetail struct { + gmeta.Meta `orm:"table:user_detail_four_layers"` + Uid int `json:"uid"` + Address string `json:"address"` + } + + type User struct { + gmeta.Meta `orm:"table:user_four_layers"` + Id int `json:"id"` + Name string `json:"name"` + UserDetail *UserDetail `orm:"with:uid=id, batch:threshold=5,batchSize=5"` + UserScores []*UserScores `orm:"with:uid=id,batch:threshold=10,batchSize=10"` + } + + // 初始化表结构 + dropTable(tableUser) + dropTable(tableUserDetail) + dropTable(tableUserScores) + dropTable(tableUserScoreDetails) + dropTable(tableDetailMeta) + + _, err := db.Exec(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id int(10) unsigned NOT NULL AUTO_INCREMENT, + name varchar(45) NOT NULL, + PRIMARY KEY (id), + KEY idx_name (name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + `, tableUser)) + gtest.AssertNil(err) + + _, err = db.Exec(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + uid int(10) unsigned NOT NULL, + address varchar(100) NOT NULL, + PRIMARY KEY (uid), + KEY idx_uid (uid) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + `, tableUserDetail)) + gtest.AssertNil(err) + + _, err = db.Exec(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id int(10) unsigned NOT NULL AUTO_INCREMENT, + uid int(10) unsigned NOT NULL, + score int(10) NOT NULL, + PRIMARY KEY (id), + KEY idx_uid (uid) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + `, tableUserScores)) + gtest.AssertNil(err) + + _, err = db.Exec(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id int(10) unsigned NOT NULL AUTO_INCREMENT, + score_id int(10) unsigned NOT NULL, + detail_info varchar(200) NOT NULL, + PRIMARY KEY (id), + KEY idx_score_id (score_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + `, tableUserScoreDetails)) + gtest.AssertNil(err) + + _, err = db.Exec(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id int(10) unsigned NOT NULL AUTO_INCREMENT, + detail_id int(10) unsigned NOT NULL, + meta_key varchar(50) NOT NULL, + meta_value varchar(100) NOT NULL, + PRIMARY KEY (id), + KEY idx_detail_id (detail_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + `, tableDetailMeta)) + gtest.AssertNil(err) + + defer dropTable(tableUser) + defer dropTable(tableUserDetail) + defer dropTable(tableUserScores) + defer dropTable(tableUserScoreDetails) + defer dropTable(tableDetailMeta) + + // ======================================== + // 数据初始化（中小数据量） + // ======================================== + const ( + userCount = 20 // 20个用户 + scorePerUser = 5 // 每个用户5个score + detailPerScore = 4 // 每个score 4个detail + metaPerDetail = 3 // 每个detail 3个meta + totalScores = userCount * scorePerUser // 100 + totalDetails = totalScores * detailPerScore // 400 + totalMeta = totalDetails * metaPerDetail // 1200 + ) + + fmt.Println("\n========== 开始初始化中小型四层数据集 ==========") + fmt.Printf("数据规模：%d 用户, %d scores, %d details, %d meta\n", userCount, totalScores, totalDetails, totalMeta) + + // 1. 插入用户数据 + fmt.Println("→ 插入用户数据...") + startTime := time.Now() + usersData := make([]*User, 0, userCount) + for i := 1; i <= userCount; i++ { + usersData = append(usersData, &User{ + Id: i, + Name: fmt.Sprintf("user_%d", i), + }) + } + _, err = db.Model(tableUser).Data(usersData).Insert() + gtest.AssertNil(err) + fmt.Printf(" 用户数据插入完成，耗时: %v\n", time.Since(startTime)) + + // 2. 插入用户详情 + fmt.Println("→ 插入用户详情...") + startTime = time.Now() + detailsData := make([]*UserDetail, 0, userCount) + for i := 1; i <= userCount; i++ { + detailsData = append(detailsData, &UserDetail{ + Uid: i, + Address: fmt.Sprintf("address_%d", i), + }) + } + _, err = db.Model(tableUserDetail).Data(detailsData).Insert() + gtest.AssertNil(err) + fmt.Printf(" 用户详情插入完成，耗时: %v\n", time.Since(startTime)) + + // 3. 插入 UserScores + fmt.Println("→ 插入 UserScores...") + startTime = time.Now() + scoresData := make([]*UserScores, 0, totalScores) + scoreId := 1 + for i := 1; i <= userCount; i++ { + for j := 1; j <= scorePerUser; j++ { + scoresData = append(scoresData, &UserScores{ + Id: scoreId, + Uid: i, + Score: j * 10, + }) + scoreId++ + } + } + _, err = db.Model(tableUserScores).Data(scoresData).Insert() + gtest.AssertNil(err) + fmt.Printf(" UserScores 插入完成，耗时: %v\n", time.Since(startTime)) + + // 4. 插入 ScoreDetails + fmt.Println("→ 插入 ScoreDetails...") + startTime = time.Now() + scoreDetailsData := make([]*UserScoreDetails, 0, totalDetails) + detailId := 1 + for i := 1; i <= totalScores; i++ { + for j := 1; j <= detailPerScore; j++ { + scoreDetailsData = append(scoreDetailsData, &UserScoreDetails{ + Id: detailId, + ScoreId: i, + DetailInfo: fmt.Sprintf("detail_%d_%d", i, j), + }) + detailId++ + } + } + _, err = db.Model(tableUserScoreDetails).Data(scoreDetailsData).Insert() + gtest.AssertNil(err) + fmt.Printf(" ScoreDetails 插入完成，耗时: %v\n", time.Since(startTime)) + + // 5. 插入 DetailMeta（第四层） + fmt.Println("→ 插入 DetailMeta...") + startTime = time.Now() + metaData := make([]*DetailMeta, 0, totalMeta) + for i := 1; i <= totalDetails; i++ { + for j := 1; j <= metaPerDetail; j++ { + metaData = append(metaData, &DetailMeta{ + DetailId: i, + MetaKey: fmt.Sprintf("key_%d", j), + MetaValue: fmt.Sprintf("value_%d_%d", i, j), + }) + } + } + _, err = db.Model(tableDetailMeta).Data(metaData).Insert() + gtest.AssertNil(err) + fmt.Printf(" DetailMeta 插入完成，耗时: %v\n", time.Since(startTime)) + + fmt.Println("========== 数据初始化完成 ==========") + + // 数据验证辅助函数 + verifyUserData := func(users []*User, expectedCount int, scenario string) { + gtest.Assert(len(users), expectedCount) + for _, user := range users { + // 验证 UserDetail + gtest.AssertNE(user.UserDetail, nil) + gtest.Assert(user.UserDetail.Address, fmt.Sprintf("address_%d", user.Id)) + + // 验证 UserScores + gtest.Assert(len(user.UserScores), scorePerUser) + for idx, score := range user.UserScores { + gtest.Assert(score.Uid, user.Id) + gtest.Assert(score.Score, (idx+1)*10) + + // 验证 ScoreDetails + gtest.Assert(len(score.ScoreDetails), detailPerScore) + for detailIdx, detail := range score.ScoreDetails { + gtest.Assert(detail.ScoreId, score.Id) + expectedInfo := fmt.Sprintf("detail_%d_%d", score.Id, detailIdx+1) + gtest.Assert(detail.DetailInfo, expectedInfo) + + // 验证 DetailMeta（第四层） + gtest.Assert(len(detail.DetailMeta), metaPerDetail) + for metaIdx, meta := range detail.DetailMeta { + gtest.Assert(meta.DetailId, detail.Id) + gtest.Assert(meta.MetaKey, fmt.Sprintf("key_%d", metaIdx+1)) + expectedValue := fmt.Sprintf("value_%d_%d", detail.Id, metaIdx+1) + gtest.Assert(meta.MetaValue, expectedValue) + } + } + } + } + fmt.Printf("✓ %s - 数据验证通过（验证了 %d 个用户的四层完整数据）\n", scenario, expectedCount) + } + + gtest.C(t, func(t *gtest.T) { + + db.SetDebug(true) + fmt.Println("\n开始执行查询...") + startTime := time.Now() + var users []*User + err := db.Model(tableUser).WithAll().Scan(&users) + duration := time.Since(startTime) + db.SetDebug(false) + + t.AssertNil(err) + fmt.Printf("\n查询完成,耗时: %v\n", duration) + + verifyUserData(users, 20, "Scenario 1") + }) + + gtest.C(t, func(t *gtest.T) { + + db.SetDebug(true) + fmt.Println("\n开始执行查询...") + startTime := time.Now() + var users []*User + err := db.Model(tableUser).WithBatch().WithAll().Scan(&users) + duration := time.Since(startTime) + db.SetDebug(false) + + t.AssertNil(err) + fmt.Printf("\n查询完成,耗时: %v\n", duration) + + verifyUserData(users, 20, "Scenario 2") + }) + fmt.Println("\n========== 中小型数据集测试全部完成 ==========") +} diff --git a/database/gdb/gdb_func.go b/database/gdb/gdb_func.go index 437c981f8fe..af5fa8e7973 100644 --- a/database/gdb/gdb_func.go +++ b/database/gdb/gdb_func.go @@ -59,13 +59,15 @@ type iTableName interface { } const ( - OrmTagForStruct = "orm" - OrmTagForTable = "table" - OrmTagForWith = "with" - OrmTagForWithWhere = "where" - OrmTagForWithOrder = "order" - OrmTagForWithUnscoped = "unscoped" - OrmTagForDo = "do" + OrmTagForStruct = "orm" + OrmTagForTable = "table" + OrmTagForWith = "with" + OrmTagForWithWhere = "where" + OrmTagForWithOrder = "order" + OrmTagForWithUnscoped = "unscoped" + OrmTagForWithBatchSize = "batchSize" + OrmTagForWithBatchThreshold = "batchThreshold" + OrmTagForDo = "do" ) var ( diff --git a/database/gdb/gdb_model.go b/database/gdb/gdb_model.go index 39b9fed3580..93ce25c0312 100644 --- a/database/gdb/gdb_model.go +++ b/database/gdb/gdb_model.go @@ -17,45 +17,46 @@ import ( // Model is core struct implementing the DAO for ORM. type Model struct { - db DB // Underlying DB interface. - tx TX // Underlying TX interface. - rawSql string // rawSql is the raw SQL string which marks a raw SQL based Model not a table based Model. - schema string // Custom database schema. - linkType int // Mark for operation on master or slave. - tablesInit string // Table names when model initialization. - tables string // Operation table names, which can be more than one table names and aliases, like: "user", "user u", "user u, user_detail ud". - fields []any // Operation fields, multiple fields joined using char ','. - fieldsEx []any // Excluded operation fields, it here uses slice instead of string type for quick filtering. - withArray []any // Arguments for With feature. - withAll bool // Enable model association operations on all objects that have "with" tag in the struct. - extraArgs []any // Extra custom arguments for sql, which are prepended to the arguments before sql committed to underlying driver. - whereBuilder *WhereBuilder // Condition builder for where operation. - groupBy string // Used for "group by" statement. - orderBy string // Used for "order by" statement. - having []any // Used for "having..." statement. - start int // Used for "select ... start, limit ..." statement. - limit int // Used for "select ... start, limit ..." statement. - option int // Option for extra operation features. - offset int // Offset statement for some databases grammar. - partition string // Partition table partition name. - data any // Data for operation, which can be type of map/[]map/struct/*struct/string, etc. - batch int // Batch number for batch Insert/Replace/Save operations. - filter bool // Filter data and where key-value pairs according to the fields of the table. - distinct string // Force the query to only return distinct results. - lockInfo string // Lock for update or in shared lock. - cacheEnabled bool // Enable sql result cache feature, which is mainly for indicating cache duration(especially 0) usage. - cacheOption CacheOption // Cache option for query statement. - pageCacheOption []CacheOption // Cache option for paging query statement. - hookHandler HookHandler // Hook functions for model hook feature. - unscoped bool // Disables soft deleting features when select/delete operations. - safe bool // If true, it clones and returns a new model object whenever operation done; or else it changes the attribute of current model. - onDuplicate any // onDuplicate is used for on Upsert clause. - onDuplicateEx any // onDuplicateEx is used for excluding some columns on Upsert clause. - onConflict any // onConflict is used for conflict keys on Upsert clause. - tableAliasMap map[string]string // Table alias to true table name, usually used in join statements. - softTimeOption SoftTimeOption // SoftTimeOption is the option to customize soft time feature for Model. - shardingConfig ShardingConfig // ShardingConfig for database/table sharding feature. - shardingValue any // Sharding value for sharding feature. + db DB // Underlying DB interface. + tx TX // Underlying TX interface. + rawSql string // rawSql is the raw SQL string which marks a raw SQL based Model not a table based Model. + schema string // Custom database schema. + linkType int // Mark for operation on master or slave. + tablesInit string // Table names when model initialization. + tables string // Operation table names, which can be more than one table names and aliases, like: "user", "user u", "user u, user_detail ud". + fields []any // Operation fields, multiple fields joined using char ','. + fieldsEx []any // Excluded operation fields, it here uses slice instead of string type for quick filtering. + withArray []any // Arguments for With feature. + withAll bool // Enable model association operations on all objects that have "with" tag in the struct. + extraArgs []any // Extra custom arguments for sql, which are prepended to the arguments before sql committed to underlying driver. + whereBuilder *WhereBuilder // Condition builder for where operation. + groupBy string // Used for "group by" statement. + orderBy string // Used for "order by" statement. + having []any // Used for "having..." statement. + start int // Used for "select ... start, limit ..." statement. + limit int // Used for "select ... start, limit ..." statement. + option int // Option for extra operation features. + offset int // Offset statement for some databases grammar. + partition string // Partition table partition name. + data any // Data for operation, which can be type of map/[]map/struct/*struct/string, etc. + batch int // Batch number for batch Insert/Replace/Save operations. + filter bool // Filter data and where key-value pairs according to the fields of the table. + distinct string // Force the query to only return distinct results. + lockInfo string // Lock for update or in shared lock. + cacheEnabled bool // Enable sql result cache feature, which is mainly for indicating cache duration(especially 0) usage. + cacheOption CacheOption // Cache option for query statement. + pageCacheOption []CacheOption // Cache option for paging query statement. + hookHandler HookHandler // Hook functions for model hook feature. + unscoped bool // Disables soft deleting features when select/delete operations. + safe bool // If true, it clones and returns a new model object whenever operation done; or else it changes the attribute of current model. + onDuplicate any // onDuplicate is used for on Upsert clause. + onDuplicateEx any // onDuplicateEx is used for excluding some columns on Upsert clause. + onConflict any // onConflict is used for conflict keys on Upsert clause. + tableAliasMap map[string]string // Table alias to true table name, usually used in join statements. + softTimeOption SoftTimeOption // SoftTimeOption is the option to customize soft time feature for Model. + shardingConfig ShardingConfig // ShardingConfig for database/table sharding feature. + shardingValue any // Sharding value for sharding feature. + withBatchEnabled bool // withBatchEnabled enables batch recursive scanning for association operations (Solving N+1 problem). } // ModelHandler is a function that handles given Model and returns a new Model that is custom modified. diff --git a/database/gdb/gdb_model_select.go b/database/gdb/gdb_model_select.go index 7d362ac1cbd..c33a26ebd0e 100644 --- a/database/gdb/gdb_model_select.go +++ b/database/gdb/gdb_model_select.go @@ -369,13 +369,15 @@ func (m *Model) ScanAndCount(pointer any, totalCount *int, useFieldForCount bool // ScanList converts `r` to struct slice which contains other complex struct attributes. // Note that the parameter `listPointer` should be type of *[]struct/*[]*struct. // -// See Result.ScanList. +// ScanList converts `r` to struct slice which contains other complex struct attributes. +// Also see Result.ScanList. func (m *Model) ScanList(structSlicePointer any, bindToAttrName string, relationAttrNameAndFields ...string) (err error) { var result Result out, err := checkGetSliceElementInfoForScanList(structSlicePointer, bindToAttrName) if err != nil { return err } + if len(m.fields) > 0 || len(m.fieldsEx) != 0 { // There are custom fields. result, err = m.All() @@ -405,6 +407,7 @@ func (m *Model) ScanList(structSlicePointer any, bindToAttrName string, relation BindToAttrName: bindToAttrName, RelationAttrName: relationAttrName, RelationFields: relationFields, + BatchEnabled: m.withBatchEnabled, }) } diff --git a/database/gdb/gdb_model_with.go b/database/gdb/gdb_model_with.go index eaea5300b4d..f29f4d33fb1 100644 --- a/database/gdb/gdb_model_with.go +++ b/database/gdb/gdb_model_with.go @@ -8,6 +8,7 @@ package gdb import ( "database/sql" + "errors" "reflect" "github.com/gogf/gf/v2/errors/gcode" @@ -15,6 +16,7 @@ import ( "github.com/gogf/gf/v2/internal/utils" "github.com/gogf/gf/v2/os/gstructs" "github.com/gogf/gf/v2/text/gstr" + "github.com/gogf/gf/v2/util/gconv" "github.com/gogf/gf/v2/util/gutil" ) @@ -65,6 +67,21 @@ func (m *Model) WithAll() *Model { return model } +// WithBatch enables or disables the batch recursive scanning feature for association operations. +// The batch recursive scanning feature is used to solve the N+1 problem by batching multiple +// association queries into one or fewer queries. +// It is disabled by default. +// 开启或关闭关联查询的批量递归扫描功能（解决N+1问题）。 +// 默认关闭，开启后可大幅提升存在大量关联数据时的查询性能。 +func (m *Model) WithBatch(enabled ...bool) *Model { + model := m.getModel() + model.withBatchEnabled = true + if len(enabled) > 0 { + model.withBatchEnabled = enabled[0] + } + return model +} + // doWithScanStruct handles model association operations feature for single struct. func (m *Model) doWithScanStruct(pointer any) error { if len(m.withArray) == 0 && !m.withAll { @@ -104,7 +121,7 @@ func (m *Model) doWithScanStruct(pointer any) error { for _, field := range currentStructFieldMap { var ( fieldTypeStr = gstr.TrimAll(field.Type().String(), "*[]") - parsedTagOutput = m.parseWithTagInFieldStruct(field) + parsedTagOutput = parseWithTagInField(field.Field) ) if parsedTagOutput.With == "" { continue @@ -121,7 +138,6 @@ func (m *Model) doWithScanStruct(pointer any) error { } var ( model *Model - fieldKeys []string relatedSourceName = array[0] relatedTargetName = array[1] relatedTargetValue any @@ -145,19 +161,9 @@ func (m *Model) doWithScanStruct(pointer any) error { bindToReflectValue = bindToReflectValue.Addr() } - if structFields, err := gstructs.Fields(gstructs.FieldsInput{ - Pointer: field.Value, - RecursiveOption: gstructs.RecursiveOptionEmbeddedNoTag, - }); err != nil { - return err - } else { - fieldKeys = make([]string, len(structFields)) - for i, field := range structFields { - fieldKeys[i] = field.Name() - } - } // Recursively with feature checks. model = m.db.With(field.Value).Hook(m.hookHandler) + model.withBatchEnabled = m.withBatchEnabled if m.withAll { model = model.WithAll() } else { @@ -172,15 +178,16 @@ func (m *Model) doWithScanStruct(pointer any) error { if parsedTagOutput.Unscoped == "true" { model = model.Unscoped() } - // With cache feature. + // Apply cache option if enabled (for query result caching, not field metadata). if m.cacheEnabled && m.cacheOption.Name == "" { model = model.Cache(m.cacheOption) } - err = model.Fields(fieldKeys). + // Fields will be automatically determined from the struct type + err = model.Fields(field.Value). Where(relatedSourceName, relatedTargetValue). Scan(bindToReflectValue) // It ignores sql.ErrNoRows in with feature. - if err != nil && err != sql.ErrNoRows { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } } @@ -196,11 +203,24 @@ func (m *Model) doWithScanStructs(pointer any) error { if v, ok := pointer.(reflect.Value); ok { pointer = v.Interface() } - var ( err error allowedTypeStrArray = make([]string, 0) + reflectValue = reflect.ValueOf(pointer) + reflectKind = reflectValue.Kind() ) + if reflectKind == reflect.Pointer { + reflectValue = reflectValue.Elem() + reflectKind = reflectValue.Kind() + } + if reflectKind != reflect.Slice && reflectKind != reflect.Array { + return gerror.NewCodef( + gcode.CodeInvalidParameter, + `the parameter "pointer" for doWithScanStructs should be type of slice, invalid type: %v`, + reflect.TypeOf(pointer), + ) + } + currentStructFieldMap, err := gstructs.FieldMap(gstructs.FieldMapInput{ Pointer: pointer, PriorityTagArray: nil, @@ -231,9 +251,11 @@ func (m *Model) doWithScanStructs(pointer any) error { for fieldName, field := range currentStructFieldMap { var ( - fieldTypeStr = gstr.TrimAll(field.Type().String(), "*[]") - parsedTagOutput = m.parseWithTagInFieldStruct(field) + fieldTypeStr = gstr.TrimAll(field.Type().String(), "*[]") ) + // Parse withTag directly from field instead of using cache to avoid cache pollution + // when multiple tests define struct with same name but different tags + parsedTagOutput := parseWithTagInField(field.Field) if parsedTagOutput.With == "" { continue } @@ -242,13 +264,10 @@ func (m *Model) doWithScanStructs(pointer any) error { } array := gstr.SplitAndTrim(parsedTagOutput.With, "=") if len(array) == 1 { - // It supports using only one column name - // if both tables associates using the same column name. array = append(array, parsedTagOutput.With) } var ( model *Model - fieldKeys []string relatedSourceName = array[0] relatedTargetName = array[1] relatedTargetValue any @@ -269,21 +288,11 @@ func (m *Model) doWithScanStructs(pointer any) error { } // If related value is empty, it does nothing but just returns. if gutil.IsEmpty(relatedTargetValue) { - return nil - } - if structFields, err := gstructs.Fields(gstructs.FieldsInput{ - Pointer: field.Value, - RecursiveOption: gstructs.RecursiveOptionEmbeddedNoTag, - }); err != nil { - return err - } else { - fieldKeys = make([]string, len(structFields)) - for i, field := range structFields { - fieldKeys[i] = field.Name() - } + continue } // Recursively with feature checks. model = m.db.With(field.Value).Hook(m.hookHandler) + model.withBatchEnabled = m.withBatchEnabled if m.withAll { model = model.WithAll() } else { @@ -298,52 +307,133 @@ func (m *Model) doWithScanStructs(pointer any) error { if parsedTagOutput.Unscoped == "true" { model = model.Unscoped() } - // With cache feature. + // Apply cache option if enabled (for query result caching, not field metadata). if m.cacheEnabled && m.cacheOption.Name == "" { model = model.Cache(m.cacheOption) } - err = model.Fields(fieldKeys). - Where(relatedSourceName, relatedTargetValue). - ScanList(pointer, fieldName, parsedTagOutput.With) - // It ignores sql.ErrNoRows in with feature. - if err != nil && err != sql.ErrNoRows { + + var ( + batchSize int + batchThreshold int + results Result + ) + + if m.withBatchEnabled { + batchSize = parsedTagOutput.BatchSize + batchThreshold = parsedTagOutput.BatchThreshold + } + + if m.withBatchEnabled && batchSize > 0 && len(gconv.SliceAny(relatedTargetValue)) >= batchThreshold { + var ids = gconv.SliceAny(relatedTargetValue) + for i := 0; i < len(ids); i += batchSize { + end := i + batchSize + if end > len(ids) { + end = len(ids) + } + // 使用 Clone() 避免条件累加 + // Fields will be automatically determined from the struct type + result, err := model.Clone().Fields(field.Value). + Where(relatedSourceName, ids[i:end]). + All() + if err != nil { + return err + } + results = append(results, result...) + } + } else { + // Fields will be automatically determined from the struct type + results, err = model.Clone().Fields(field.Value). + Where(relatedSourceName, relatedTargetValue). + All() + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + } + + if results.IsEmpty() { + continue + } + + err = doScanList(doScanListInput{ + Model: model, + Result: results, + StructSlicePointer: pointer, + StructSliceValue: reflect.ValueOf(pointer).Elem(), + BindToAttrName: fieldName, + RelationAttrName: "", + RelationFields: parsedTagOutput.With, + BatchEnabled: m.withBatchEnabled, + BatchSize: batchSize, + BatchThreshold: batchThreshold, + }) + if err != nil { return err } } return nil } -type parseWithTagInFieldStructOutput struct { - With string - Where string - Order string - Unscoped string +type withTagOutput struct { + With string + Where string + Order string + Unscoped string + BatchSize int + BatchThreshold int } -func (m *Model) parseWithTagInFieldStruct(field gstructs.Field) (output parseWithTagInFieldStructOutput) { +func parseWithTagInField(field reflect.StructField) (output withTagOutput) { var ( - ormTag = field.Tag(OrmTagForStruct) + ormTag = field.Tag.Get(OrmTagForStruct) data = make(map[string]string) - array []string - key string ) + // Parse tags, support key:value and nested batch:threshold=1000,batchSize=100 for _, v := range gstr.SplitAndTrim(ormTag, ",") { - array = gstr.Split(v, ":") - if len(array) == 2 { - key = array[0] - data[key] = gstr.Trim(array[1]) + v = gstr.Trim(v) + if v == "" { + continue + } + + // 处理 batch: 开头的特殊配置 + if gstr.HasPrefix(v, "batch:") { + // 提取 batch: 后面的内容 + batchConfig := gstr.TrimLeft(v, "batch:") + // 解析 batch 内部的配置项（如 threshold=1000,batchSize=100） + for _, batchItem := range gstr.SplitAndTrim(batchConfig, ",") { + parts := gstr.Split(batchItem, "=") + if len(parts) == 2 { + data[gstr.Trim(parts[0])] = gstr.Trim(parts[1]) + } + } + continue + } + + // Process normal key:value or key=value + var ( + key string + value string + parts = gstr.Split(v, ":") + ) + if len(parts) == 2 { + key = gstr.Trim(parts[0]) + value = gstr.Trim(parts[1]) } else { - if key == OrmTagForWithOrder { - // supporting multiple order fields - data[key] += "," + gstr.Trim(v) - } else { - data[key] += " " + gstr.Trim(v) + parts = gstr.Split(v, "=") + if len(parts) == 2 { + key = gstr.Trim(parts[0]) + value = gstr.Trim(parts[1]) } } + + if key != "" { + data[key] = value + } } output.With = data[OrmTagForWith] output.Where = data[OrmTagForWithWhere] output.Order = data[OrmTagForWithOrder] output.Unscoped = data[OrmTagForWithUnscoped] + output.BatchSize = gconv.Int(data[OrmTagForWithBatchSize]) + output.BatchThreshold = gconv.Int(data[OrmTagForWithBatchThreshold]) return } diff --git a/database/gdb/gdb_type_result_scanlist.go b/database/gdb/gdb_type_result_scanlist.go index 0017c090b2f..91d4482f1a8 100644 --- a/database/gdb/gdb_type_result_scanlist.go +++ b/database/gdb/gdb_type_result_scanlist.go @@ -116,12 +116,16 @@ func (r Result) ScanList(structSlicePointer any, bindToAttrName string, relation BindToAttrName: bindToAttrName, RelationAttrName: relationAttrName, RelationFields: relationFields, + BatchEnabled: false, + BatchSize: 0, + BatchThreshold: 0, }) } type checkGetSliceElementInfoForScanListOutput struct { SliceReflectValue reflect.Value BindToAttrType reflect.Type + BindToAttrField reflect.StructField } func checkGetSliceElementInfoForScanList(structSlicePointer any, bindToAttrName string) (out *checkGetSliceElementInfoForScanListOutput, err error) { @@ -176,6 +180,7 @@ func checkGetSliceElementInfoForScanList(structSlicePointer any, bindToAttrName reflect.TypeOf(structSlicePointer).String(), ) } + out.BindToAttrField = structField // Find the attribute struct type for ORM fields filtering. reflectType = structField.Type reflectKind = reflectType.Kind() @@ -199,6 +204,23 @@ type doScanListInput struct { BindToAttrName string RelationAttrName string RelationFields string + BatchEnabled bool + BatchSize int + BatchThreshold int +} + +// doScanListRelation is the relation metadata for doScanList. +type doScanListRelation struct { + DataMap map[string]Value // Relation data map, which is Map[RelationValue]Record/Result. + FromFieldName string // The field name of the result that is used for relation. + BindToFieldName string // The attribute name of the struct that is used for relation. +} + +// doScanListBindAttr is the binding attribute information for doScanList. +type doScanListBindAttr struct { + Field reflect.StructField // The struct field of the attribute. + Kind reflect.Kind // The kind of the attribute. + Type reflect.Type // The type of the attribute. } // doScanList converts `result` to struct slice which contains other complex struct attributes recursively. @@ -213,300 +235,534 @@ func doScanList(in doScanListInput) (err error) { return gerror.NewCode(gcode.CodeInvalidParameter, `bindToAttrName should not be empty`) } - length := len(in.Result) + var ( + length = len(in.Result) + arrayValue reflect.Value + arrayItemType reflect.Type + reflectType = reflect.TypeOf(in.StructSlicePointer) + ) if length == 0 { - // The pointed slice is not empty. if in.StructSliceValue.Len() > 0 { - // It here checks if it has struct item, which is already initialized. - // It then returns error to warn the developer its empty and no conversion. if v := in.StructSliceValue.Index(0); v.Kind() != reflect.Pointer { return sql.ErrNoRows } } - // Do nothing for empty struct slice. return nil } - var ( - arrayValue reflect.Value // Like: []*Entity - arrayItemType reflect.Type // Like: *Entity - reflectType = reflect.TypeOf(in.StructSlicePointer) - ) + if in.StructSliceValue.Len() > 0 { arrayValue = in.StructSliceValue } else { arrayValue = reflect.MakeSlice(reflectType.Elem(), length, length) } - - // Slice element item. arrayItemType = arrayValue.Index(0).Type() - // Relation variables. - var ( - relationDataMap map[string]Value - relationFromFieldName string // Eg: relationKV: id:uid -> id - relationBindToFieldName string // Eg: relationKV: id:uid -> uid + // 1. Parse relation metadata. + relation, err := doScanListParseRelation(in) + if err != nil { + return err + } + + // 2. Get target attribute info. + attr, err := doScanListGetBindAttrInfo(arrayItemType, in.BindToAttrName) + if err != nil { + return err + } + + // Use field cache manager to get deterministic field access metadata. + // This caches field indices and type information (NOT tag semantics) to avoid repeated reflection in loops. + cacheItem, err := fieldCacheInstance.getOrSet( + arrayItemType, + in.BindToAttrName, + in.RelationAttrName, ) + if err != nil { + return err + } + + // 3. Batch recursive scanning optimization. + // Batch recursive scanning pre-fetches all child IDs and performs bulk queries to solve the N+1 performance problem. + // Note: BatchSize and BatchThreshold are passed via doScanListInput instead of cache + // because they may vary per query (e.g., different WithBatchOption configurations). + structsMap, err := doScanListGetBatchRecursiveMap(in, attr, relation, cacheItem) + if err != nil { + return err + } + + // 4. Final assignment loop. + // Final assignment loop: Distribute queried data to various struct attributes. + if err = doScanListAssignmentLoop(in, arrayValue, attr, &relation, structsMap, cacheItem); err != nil { + return err + } + + reflect.ValueOf(in.StructSlicePointer).Elem().Set(arrayValue) + return nil +} + +// doScanListParseRelation parses the relation metadata from input. +func doScanListParseRelation(in doScanListInput) (relation doScanListRelation, err error) { if len(in.RelationFields) > 0 { - // The relation key string of table field name and attribute name - // can be joined with char '=' or ':'. array := gstr.SplitAndTrim(in.RelationFields, "=") if len(array) == 1 { - // Compatible with old splitting char ':'. array = gstr.SplitAndTrim(in.RelationFields, ":") } if len(array) == 1 { - // The relation names are the same. array = []string{in.RelationFields, in.RelationFields} } if len(array) == 2 { - // Defined table field to relation attribute name. - // Like: - // uid:Uid - // uid:UserId - relationFromFieldName = array[0] - relationBindToFieldName = array[1] - if key, _ := gutil.MapPossibleItemByKey(in.Result[0].Map(), relationFromFieldName); key == "" { - return gerror.NewCodef( + relation.FromFieldName = array[0] + relation.BindToFieldName = array[1] + if key, _ := gutil.MapPossibleItemByKey(in.Result[0].Map(), relation.FromFieldName); key == "" { + return relation, gerror.NewCodef( gcode.CodeInvalidParameter, `cannot find possible related table field name "%s" from given relation fields "%s"`, - relationFromFieldName, + relation.FromFieldName, in.RelationFields, ) } else { - relationFromFieldName = key + relation.FromFieldName = key } } else { - return gerror.NewCode( + return relation, gerror.NewCode( gcode.CodeInvalidParameter, `parameter relationKV should be format of "ResultFieldName:BindToAttrName"`, ) } - if relationFromFieldName != "" { - // Note that the value might be type of slice. - relationDataMap = in.Result.MapKeyValue(relationFromFieldName) + if relation.FromFieldName != "" { + relation.DataMap = in.Result.MapKeyValue(relation.FromFieldName) } - if len(relationDataMap) == 0 { - return gerror.NewCodef( + if len(relation.DataMap) == 0 { + return relation, gerror.NewCodef( gcode.CodeInvalidParameter, `cannot find the relation data map, maybe invalid relation fields given "%v"`, in.RelationFields, ) } } - // Bind to target attribute. - var ( - ok bool - bindToAttrValue reflect.Value - bindToAttrKind reflect.Kind - bindToAttrType reflect.Type - bindToAttrField reflect.StructField - ) + return relation, nil +} + +// doScanListGetBindAttrInfo gets the binding attribute information from given array item type and name. +func doScanListGetBindAttrInfo(arrayItemType reflect.Type, bindToAttrName string) (attr doScanListBindAttr, err error) { + var ok bool if arrayItemType.Kind() == reflect.Pointer { - if bindToAttrField, ok = arrayItemType.Elem().FieldByName(in.BindToAttrName); !ok { - return gerror.NewCodef( + if attr.Field, ok = arrayItemType.Elem().FieldByName(bindToAttrName); !ok { + return attr, gerror.NewCodef( gcode.CodeInvalidParameter, `invalid parameter bindToAttrName: cannot find attribute with name "%s" from slice element`, - in.BindToAttrName, + bindToAttrName, ) } } else { - if bindToAttrField, ok = arrayItemType.FieldByName(in.BindToAttrName); !ok { - return gerror.NewCodef( + if attr.Field, ok = arrayItemType.FieldByName(bindToAttrName); !ok { + return attr, gerror.NewCodef( gcode.CodeInvalidParameter, `invalid parameter bindToAttrName: cannot find attribute with name "%s" from slice element`, - in.BindToAttrName, + bindToAttrName, ) } } - bindToAttrType = bindToAttrField.Type - bindToAttrKind = bindToAttrType.Kind() + attr.Type = attr.Field.Type + attr.Kind = attr.Type.Kind() + return attr, nil +} + +// doScanListGetBatchRecursiveMap executes the batch recursive scanning optimization (Solving N+1 problem). +// It returns a map that contains the relational structs, which can be used for fast assignment in the loop. +// Core logic for batch recursive scanning: +// 1. Get batch configuration (BatchSize, BatchThreshold). +// 2. Chunk scan all records in Result to a temporary slice. +// 3. Build a map with relation field as Key for subsequent O(1) complexity assignment loop. +// 4. Perform recursive association queries on the temporary slice (doWithScanStructs). +func doScanListGetBatchRecursiveMap( + in doScanListInput, attr doScanListBindAttr, relation doScanListRelation, cacheItem *fieldCacheItem, +) (relationStructsMap map[string]reflect.Value, err error) { + if !in.BatchEnabled || len(in.Result) < in.BatchThreshold { + return nil, nil + } + + if in.Model != nil && len(in.Result) > 0 { + var ( + allChildStructsSlice reflect.Value + batchSize = in.BatchSize + ) + if batchSize <= 0 { + batchSize = 1000 + } + + // Step 1: Prepare the container for bulk scanning. + if attr.Kind == reflect.Array || attr.Kind == reflect.Slice { + allChildStructsSlice = reflect.MakeSlice(attr.Field.Type, 0, len(in.Result)) + } else { + allChildStructsSlice = reflect.MakeSlice(reflect.SliceOf(attr.Field.Type), 0, len(in.Result)) + } + + // Step 2: Extract all child records from relation.DataMap and scan them into a single slice. + // relation.DataMap structure: map[parentKey][]Record (e.g., map["1"][]UserScore records for uid=1) + if relation.FromFieldName != "" && len(relation.DataMap) > 0 { + // Collect all child records from DataMap + allChildRecords := make(Result, 0) + for _, records := range relation.DataMap { + for _, record := range records.Slice() { + allChildRecords = append(allChildRecords, record.(Record)) + } + } + + // Scan all child records into the target slice type + if len(allChildRecords) > 0 { + if attr.Kind == reflect.Array || attr.Kind == reflect.Slice { + allChildStructsPtr := reflect.New(attr.Field.Type).Interface() + if err = allChildRecords.Structs(allChildStructsPtr); err != nil { + return nil, err + } + allChildStructsSlice = reflect.ValueOf(allChildStructsPtr).Elem() + } else { + // For non-slice types (pointer), we still need to process all records + // but will only use the first match per key in Step 4 + allChildStructsPtr := reflect.New(reflect.SliceOf(attr.Field.Type)).Interface() + if err = allChildRecords.Structs(allChildStructsPtr); err != nil { + return nil, err + } + allChildStructsSlice = reflect.ValueOf(allChildStructsPtr).Elem() + } + } + } + + // Step 3: Execute recursive relation queries for ALL records at once (Breadth-First). + // We create an addressable pointer for the entire slice to allow doWithScanStructs to bind results back. + allChildStructsSlicePtr := reflect.New(allChildStructsSlice.Type()) + allChildStructsSlicePtr.Elem().Set(allChildStructsSlice) + if err = in.Model.doWithScanStructs(allChildStructsSlicePtr.Interface()); err != nil { + return nil, err + } + // Sync back the results if they were modified (e.g., pointers filled). + allChildStructsSlice = allChildStructsSlicePtr.Elem() + + // Step 4: Build a map for fast lookup in the main assignment loop. + if relation.FromFieldName != "" { + relationStructsMap = make(map[string]reflect.Value) + for i := 0; i < allChildStructsSlice.Len(); i++ { + // Extract the key from the struct field directly, not from in.Result + // because doWithScanStructs may have filtered some records (e.g., where conditions) + structItem := allChildStructsSlice.Index(i) + if structItem.Kind() == reflect.Pointer { + // Check if pointer is nil (filtered by where condition) + if structItem.IsNil() { + continue + } + structItem = structItem.Elem() + } + + // Get the relation field value from the struct + // In batch mode, we need to use FromFieldName (the child table's field) as the map key + // For example: UserScores.uid -> map["1"] (where "1" is the uid value) + relationFieldValue := structItem.FieldByName(relation.FromFieldName) + if !relationFieldValue.IsValid() { + // Try case-insensitive lookup using gstructs.FieldMap + fieldMap, _ := gstructs.FieldMap(gstructs.FieldMapInput{ + Pointer: structItem, + RecursiveOption: gstructs.RecursiveOptionEmbeddedNoTag, + }) + if key, _ := gutil.MapPossibleItemByKey(gconv.Map(fieldMap), relation.FromFieldName); key != "" { + relationFieldValue = structItem.FieldByName(key) + } + } + if !relationFieldValue.IsValid() { + continue + } + + kv := gconv.String(relationFieldValue.Interface()) + if attr.Kind == reflect.Array || attr.Kind == reflect.Slice { + if _, ok := relationStructsMap[kv]; !ok { + relationStructsMap[kv] = reflect.MakeSlice(attr.Field.Type, 0, 0) + } + relationStructsMap[kv] = reflect.Append(relationStructsMap[kv], allChildStructsSlice.Index(i)) + } else { + if _, ok := relationStructsMap[kv]; !ok { + relationStructsMap[kv] = allChildStructsSlice.Index(i) + } + } + } + } + } + return relationStructsMap, nil +} - // Bind to relation conditions. +// doScanListAssignmentLoop executes the final assignment loop for ScanList. +func doScanListAssignmentLoop( + in doScanListInput, + arrayValue reflect.Value, + attr doScanListBindAttr, + relation *doScanListRelation, + structsMap map[string]reflect.Value, + cacheItem *fieldCacheItem, +) (err error) { var ( + arrayItemType = arrayValue.Index(0).Type() relationFromAttrValue reflect.Value relationFromAttrField reflect.Value relationBindToFieldNameChecked bool ) + for i := 0; i < arrayValue.Len(); i++ { arrayElemValue := arrayValue.Index(i) - // The FieldByName should be called on non-pointer reflect.Value. - if arrayElemValue.Kind() == reflect.Pointer { - // Like: []*Entity + + // Use cached type information (pointer element check) + if cacheItem.isPointerElem { arrayElemValue = arrayElemValue.Elem() if !arrayElemValue.IsValid() { - // The element is nil, then create one and set it to the slice. - // The "reflect.New(itemType.Elem())" creates a new element and returns the address of it. - // For example: - // reflect.New(itemType.Elem()) => *Entity - // reflect.New(itemType.Elem()).Elem() => Entity arrayElemValue = reflect.New(arrayItemType.Elem()).Elem() arrayValue.Index(i).Set(arrayElemValue.Addr()) } - // } else { - // Like: []Entity } - bindToAttrValue = arrayElemValue.FieldByName(in.BindToAttrName) - if in.RelationAttrName != "" { - // Attribute value of current slice element. - relationFromAttrValue = arrayElemValue.FieldByName(in.RelationAttrName) + + // Use cached field index for direct access (avoid FieldByName) + var bindToAttrValue reflect.Value + if cacheItem.bindToAttrIndex >= 0 { + // Direct field access + bindToAttrValue = arrayElemValue.Field(cacheItem.bindToAttrIndex) + } else if len(cacheItem.bindToAttrIndexPath) > 0 { + // Embedded field access using index path + bindToAttrValue = arrayElemValue.FieldByIndex(cacheItem.bindToAttrIndexPath) + } else { + // Fallback to FieldByName (shouldn't happen if cache is correct) + bindToAttrValue = arrayElemValue.FieldByName(in.BindToAttrName) + } + + // Get relation attribute value + if cacheItem.relationAttrIndex >= 0 { + relationFromAttrValue = arrayElemValue.Field(cacheItem.relationAttrIndex) + if relationFromAttrValue.Kind() == reflect.Pointer { + relationFromAttrValue = relationFromAttrValue.Elem() + } + } else if len(cacheItem.relationAttrIndexPath) > 0 { + // Embedded field access using index path + relationFromAttrValue = arrayElemValue.FieldByIndex(cacheItem.relationAttrIndexPath) if relationFromAttrValue.Kind() == reflect.Pointer { relationFromAttrValue = relationFromAttrValue.Elem() } } else { - // Current slice element. relationFromAttrValue = arrayElemValue } - if len(relationDataMap) > 0 && !relationFromAttrValue.IsValid() { + + if len(relation.DataMap) > 0 && !relationFromAttrValue.IsValid() { return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) } - // Check and find possible bind to attribute name. + + // Dynamic lookup for embedded fields (NOT cached due to runtime dependency) if in.RelationFields != "" && !relationBindToFieldNameChecked { - relationFromAttrField = relationFromAttrValue.FieldByName(relationBindToFieldName) + relationFromAttrField = relationFromAttrValue.FieldByName(relation.BindToFieldName) if !relationFromAttrField.IsValid() { fieldMap, _ := gstructs.FieldMap(gstructs.FieldMapInput{ Pointer: relationFromAttrValue, RecursiveOption: gstructs.RecursiveOptionEmbeddedNoTag, }) - if key, _ := gutil.MapPossibleItemByKey(gconv.Map(fieldMap), relationBindToFieldName); key == "" { + if key, _ := gutil.MapPossibleItemByKey(gconv.Map(fieldMap), relation.BindToFieldName); key == "" { return gerror.NewCodef( gcode.CodeInvalidParameter, `cannot find possible related attribute name "%s" from given relation fields "%s"`, - relationBindToFieldName, + relation.BindToFieldName, in.RelationFields, ) } else { - relationBindToFieldName = key + relation.BindToFieldName = key } } relationBindToFieldNameChecked = true } - switch bindToAttrKind { + + // Dispatch based on cached attribute type + switch attr.Kind { case reflect.Array, reflect.Slice: - if len(relationDataMap) > 0 { - relationFromAttrField = relationFromAttrValue.FieldByName(relationBindToFieldName) - if relationFromAttrField.IsValid() { - results := make(Result, 0) - for _, v := range relationDataMap[gconv.String(relationFromAttrField.Interface())].Slice() { - results = append(results, v.(Record)) - } - if err = results.Structs(bindToAttrValue.Addr()); err != nil { - return err - } - // Recursively Scan. - if in.Model != nil { - if err = in.Model.doWithScanStructs(bindToAttrValue.Addr()); err != nil { - return nil - } - } - } else { - // Maybe the attribute does not exist yet. - return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) - } - } else { - return gerror.NewCodef( - gcode.CodeInvalidParameter, - `relationKey should not be empty as field "%s" is slice`, - in.BindToAttrName, - ) + if err = doScanListHandleAssignmentSlice(in, bindToAttrValue, relationFromAttrValue, *relation, structsMap); err != nil { + return err } case reflect.Pointer: - var element reflect.Value - if bindToAttrValue.IsNil() { - element = reflect.New(bindToAttrType.Elem()).Elem() - } else { - element = bindToAttrValue.Elem() + if err = doScanListHandleAssignmentPointer(in, bindToAttrValue, relationFromAttrValue, *relation, structsMap, attr, i); err != nil { + return err } - if len(relationDataMap) > 0 { - relationFromAttrField = relationFromAttrValue.FieldByName(relationBindToFieldName) - if relationFromAttrField.IsValid() { - v := relationDataMap[gconv.String(relationFromAttrField.Interface())] - if v == nil { - // There's no relational data. - continue - } - if v.IsSlice() { - if err = v.Slice()[0].(Record).Struct(element); err != nil { - return err - } - } else { - if err = v.Val().(Record).Struct(element); err != nil { - return err - } - } - } else { - // Maybe the attribute does not exist yet. - return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) - } - } else { - if i >= len(in.Result) { - // There's no relational data. - continue - } - v := in.Result[i] - if v == nil { - // There's no relational data. - continue - } - if err = v.Struct(element); err != nil { - return err - } + + case reflect.Struct: + if err = doScanListHandleAssignmentStruct(in, bindToAttrValue, relationFromAttrValue, *relation, structsMap, i); err != nil { + return err + } + + default: + return gerror.NewCodef(gcode.CodeInvalidParameter, `unsupported attribute type: %s`, attr.Kind.String()) + } + } + return nil +} + +// doScanListHandleAssignmentSlice handles the assignment for slice attribute. +func doScanListHandleAssignmentSlice( + in doScanListInput, + bindToAttrValue reflect.Value, + relationFromAttrValue reflect.Value, + relation doScanListRelation, + structsMap map[string]reflect.Value, +) error { + if len(structsMap) > 0 { + relationFromAttrField := relationFromAttrValue.FieldByName(relation.BindToFieldName) + if relationFromAttrField.IsValid() { + key := gconv.String(relationFromAttrField.Interface()) + if structs, ok := structsMap[key]; ok { + bindToAttrValue.Set(structs) + } + } else { + return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) + } + } else if len(relation.DataMap) > 0 { + relationFromAttrField := relationFromAttrValue.FieldByName(relation.BindToFieldName) + if relationFromAttrField.IsValid() { + results := make(Result, 0) + for _, v := range relation.DataMap[gconv.String(relationFromAttrField.Interface())].Slice() { + results = append(results, v.(Record)) + } + if err := results.Structs(bindToAttrValue.Addr()); err != nil { + return err } - // Recursively Scan. if in.Model != nil { - if err = in.Model.doWithScanStruct(element); err != nil { + if err := in.Model.doWithScanStructs(bindToAttrValue.Addr()); err != nil { return err } } - bindToAttrValue.Set(element.Addr()) + } else { + return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) + } + } else { + return gerror.NewCodef( + gcode.CodeInvalidParameter, + `relationKey should not be empty as field "%s" is slice`, + in.BindToAttrName, + ) + } + return nil +} - case reflect.Struct: - if len(relationDataMap) > 0 { - relationFromAttrField = relationFromAttrValue.FieldByName(relationBindToFieldName) - if relationFromAttrField.IsValid() { - relationDataItem := relationDataMap[gconv.String(relationFromAttrField.Interface())] - if relationDataItem == nil { - // There's no relational data. - continue - } - if relationDataItem.IsSlice() { - if err = relationDataItem.Slice()[0].(Record).Struct(bindToAttrValue); err != nil { - return err - } - } else { - if err = relationDataItem.Val().(Record).Struct(bindToAttrValue); err != nil { - return err - } - } - } else { - // Maybe the attribute does not exist yet. - return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) +// doScanListHandleAssignmentPointer handles the assignment for pointer attribute. +func doScanListHandleAssignmentPointer( + in doScanListInput, + bindToAttrValue reflect.Value, + relationFromAttrValue reflect.Value, + relation doScanListRelation, + structsMap map[string]reflect.Value, + attr doScanListBindAttr, + index int, +) error { + var element reflect.Value + if bindToAttrValue.Kind() == reflect.Pointer && bindToAttrValue.IsNil() { + element = reflect.New(attr.Type.Elem()).Elem() + } else { + element = bindToAttrValue.Elem() + } + if len(structsMap) > 0 { + relationFromAttrField := relationFromAttrValue.FieldByName(relation.BindToFieldName) + if relationFromAttrField.IsValid() { + key := gconv.String(relationFromAttrField.Interface()) + if structs, ok := structsMap[key]; ok { + bindToAttrValue.Set(structs) + } + } else { + return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) + } + } else if len(relation.DataMap) > 0 { + relationFromAttrField := relationFromAttrValue.FieldByName(relation.BindToFieldName) + if relationFromAttrField.IsValid() { + v := relation.DataMap[gconv.String(relationFromAttrField.Interface())] + if v == nil { + return nil + } + if v.IsSlice() { + if err := v.Slice()[0].(Record).Struct(element); err != nil { + return err } } else { - if i >= len(in.Result) { - // There's no relational data. - continue - } - relationDataItem := in.Result[i] - if relationDataItem == nil { - // There's no relational data. - continue - } - if err = relationDataItem.Struct(bindToAttrValue); err != nil { + if err := v.Val().(Record).Struct(element); err != nil { return err } } - // Recursively Scan. - if in.Model != nil { - if err = in.Model.doWithScanStruct(bindToAttrValue); err != nil { + } else { + return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) + } + } else { + if index >= len(in.Result) { + return nil + } + v := in.Result[index] + if v == nil { + return nil + } + if err := v.Struct(element); err != nil { + return err + } + } + if in.Model != nil && len(structsMap) == 0 { + if err := in.Model.doWithScanStruct(element); err != nil { + return err + } + } + if len(structsMap) == 0 { + bindToAttrValue.Set(element.Addr()) + } + return nil +} + +// doScanListHandleAssignmentStruct handles the assignment for struct attribute. +func doScanListHandleAssignmentStruct( + in doScanListInput, + bindToAttrValue reflect.Value, + relationFromAttrValue reflect.Value, + relation doScanListRelation, + structsMap map[string]reflect.Value, + index int, +) error { + if len(structsMap) > 0 { + relationFromAttrField := relationFromAttrValue.FieldByName(relation.BindToFieldName) + if relationFromAttrField.IsValid() { + key := gconv.String(relationFromAttrField.Interface()) + if structs, ok := structsMap[key]; ok { + bindToAttrValue.Set(structs) + } + } else { + return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) + } + } else if len(relation.DataMap) > 0 { + relationFromAttrField := relationFromAttrValue.FieldByName(relation.BindToFieldName) + if relationFromAttrField.IsValid() { + relationDataItem := relation.DataMap[gconv.String(relationFromAttrField.Interface())] + if relationDataItem == nil { + return nil + } + if relationDataItem.IsSlice() { + if err := relationDataItem.Slice()[0].(Record).Struct(bindToAttrValue); err != nil { + return err + } + } else { + if err := relationDataItem.Val().(Record).Struct(bindToAttrValue); err != nil { return err } } - - default: - return gerror.NewCodef(gcode.CodeInvalidParameter, `unsupported attribute type: %s`, bindToAttrKind.String()) + } else { + return gerror.NewCodef(gcode.CodeInvalidParameter, `invalid relation fields specified: "%v"`, in.RelationFields) + } + } else { + if index >= len(in.Result) { + return nil + } + relationDataItem := in.Result[index] + if relationDataItem == nil { + return nil + } + if err := relationDataItem.Struct(bindToAttrValue); err != nil { + return err + } + } + if in.Model != nil && len(structsMap) == 0 { + if err := in.Model.doWithScanStruct(bindToAttrValue); err != nil { + return err } } - reflect.ValueOf(in.StructSlicePointer).Elem().Set(arrayValue) return nil } diff --git a/database/gdb/gdb_type_result_scanlist_cache.go b/database/gdb/gdb_type_result_scanlist_cache.go new file mode 100644 index 00000000000..c6f13200f8d --- /dev/null +++ b/database/gdb/gdb_type_result_scanlist_cache.go @@ -0,0 +1,193 @@ +// Copyright GoFrame Author(https://goframe.org). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "fmt" + "reflect" + "strings" + "sync" +) + +// Field metadata cache manager +// Design principle: +// 1. Cache deterministic information only (field index, type judgment) +// 2. Retain dynamic lookup capability (embedded fields, case insensitive) +// 3. Use sync.Map to ensure high performance concurrent access + +// fieldCacheManager field cache manager +type fieldCacheManager struct { + cache sync.Map // map[fieldCacheKey]*fieldCacheItem +} + +// fieldCacheKey is the composite key for field cache +// Using struct instead of string to properly distinguish types with same name +type fieldCacheKey struct { + typ reflect.Type // The actual type (not just string representation) + bindToAttrName string + relationAttrName string +} + +// newFieldCacheManager creates field cache manager +func newFieldCacheManager() *fieldCacheManager { + return &fieldCacheManager{} +} + +// fieldCacheInstance global field cache manager instance +var fieldCacheInstance = newFieldCacheManager() + +// fieldCacheItem field cache +// Stores deterministic field information that can be safely cached to avoid repeated reflection in loops +// Note: withTag and related batch settings are NOT cached because they contain dynamic semantics +// (e.g., where/order conditions) that may differ across struct definitions with the same type name. +type fieldCacheItem struct { + // Deterministic field index (can be safely cached) + bindToAttrIndex int // Field index of bound attribute (e.g. UserDetail), -1 for embedded fields + bindToAttrIndexPath []int // Full index path for embedded fields (e.g. []int{1, 2}) + relationAttrIndex int // Field index of relation attribute (e.g. User, -1 means none) + relationAttrIndexPath []int // Full index path for embedded relation attribute + isPointerElem bool // Whether array element is pointer type + bindToAttrKind reflect.Kind // Type of bound attribute + + // Field name mapping (supports case-insensitive lookup) + fieldNameMap map[string]string // lowercase -> OriginalName + fieldIndexMap map[string]int // FieldName -> Index +} + +// getOrSet gets or sets cache (thread-safe) +func (m *fieldCacheManager) getOrSet( + arrayItemType reflect.Type, + bindToAttrName string, + relationAttrName string, +) (*fieldCacheItem, error) { + // Build cache key using reflect.Type directly + cacheKey := fieldCacheKey{ + typ: arrayItemType, + bindToAttrName: bindToAttrName, + relationAttrName: relationAttrName, + } + + // Fast path: cache hit + if cached, ok := m.cache.Load(cacheKey); ok { + return cached.(*fieldCacheItem), nil + } + + // Slow path: build cache + cache, err := m.buildCache(arrayItemType, bindToAttrName, relationAttrName) + if err != nil { + return nil, err + } + + // Store to cache (if built concurrently, only one will be saved) + actual, _ := m.cache.LoadOrStore(cacheKey, cache) + return actual.(*fieldCacheItem), nil +} + +// buildCache builds field access cache +func (m *fieldCacheManager) buildCache( + arrayItemType reflect.Type, + bindToAttrName string, + relationAttrName string, +) (*fieldCacheItem, error) { + // Get the actual struct type + structType := arrayItemType + isPointerElem := false + if structType.Kind() == reflect.Pointer { + structType = structType.Elem() + isPointerElem = true + } + + if structType.Kind() != reflect.Struct { + return nil, fmt.Errorf("arrayItemType must be struct or pointer to struct, got: %s", arrayItemType.Kind()) + } + + numField := structType.NumField() + cache := &fieldCacheItem{ + relationAttrIndex: -1, + isPointerElem: isPointerElem, + fieldNameMap: make(map[string]string, numField), // Pre-allocate capacity + fieldIndexMap: make(map[string]int, numField), // Pre-allocate capacity + } + + // Iterate all fields, build field mapping + for i := 0; i < numField; i++ { + field := structType.Field(i) + fieldName := field.Name + + cache.fieldIndexMap[fieldName] = i + cache.fieldNameMap[strings.ToLower(fieldName)] = fieldName + } + + // Find bindToAttrName field index + if idx, ok := cache.fieldIndexMap[bindToAttrName]; ok { + cache.bindToAttrIndex = idx + field := structType.Field(idx) + cache.bindToAttrKind = field.Type.Kind() + } else { + // Case-insensitive lookup + lowerBindName := strings.ToLower(bindToAttrName) + if originalName, ok := cache.fieldNameMap[lowerBindName]; ok { + cache.bindToAttrIndex = cache.fieldIndexMap[originalName] + field := structType.Field(cache.bindToAttrIndex) + cache.bindToAttrKind = field.Type.Kind() + } else { + // Try to find embedded field using FieldByName (supports anonymous/embedded struct) + field, ok := structType.FieldByName(bindToAttrName) + if !ok { + return nil, fmt.Errorf(`field "%s" not found in type %s`, bindToAttrName, arrayItemType.String()) + } + // For embedded fields, field.Index contains the full path + if len(field.Index) == 1 { + // Direct field (shouldn't happen as we already checked fieldIndexMap) + cache.bindToAttrIndex = field.Index[0] + } else { + // Embedded field - store the full index path + cache.bindToAttrIndex = -1 // Mark as embedded field + cache.bindToAttrIndexPath = field.Index + } + cache.bindToAttrKind = field.Type.Kind() + } + } + + // Find relationAttrName field index (optional) + if relationAttrName != "" { + if idx, ok := cache.fieldIndexMap[relationAttrName]; ok { + cache.relationAttrIndex = idx + } else { + // Case-insensitive lookup + lowerRelName := strings.ToLower(relationAttrName) + if originalName, ok := cache.fieldNameMap[lowerRelName]; ok { + cache.relationAttrIndex = cache.fieldIndexMap[originalName] + } else { + // Try to find embedded field + if field, ok := structType.FieldByName(relationAttrName); ok { + if len(field.Index) == 1 { + cache.relationAttrIndex = field.Index[0] + } else { + // Embedded field + cache.relationAttrIndex = -1 + cache.relationAttrIndexPath = field.Index + } + } + // Note: if still not found, keep -1, indicating that arrayElemValue itself should be used + } + } + } + + return cache, nil +} + +// clear clears all cache (used for testing or hot updates) +func (m *fieldCacheManager) clear() { + m.cache.Clear() +} + +// ClearFieldCache clears field cache (for external calls) +// Used for testing or application hot update scenarios +func ClearFieldCache() { + fieldCacheInstance.clear() +}