diff --git a/CHANGELOG.md b/CHANGELOG.md index 26d76a1fa8..ca0379b617 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [keep a changelog](http://keepachangelog.com) and this pr ### Changed - Add create_time and update_time to returned storage engine writes acks. -- Add storageIndexList indexOnly param to read data only from the index. +- Add storage index create flag to read only from the index. ### [3.17.1] - 2023-08-23 ### Added diff --git a/go.sum b/go.sum index 3501200dda..63273b4115 100644 --- a/go.sum +++ b/go.sum @@ -292,6 +292,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/heroiclabs/nakama-common v1.28.1 h1:GOH8r27NBSdiBf7xjy2R/Ov7CSr9zp4DVJt9Y2LIbqg= +github.com/heroiclabs/nakama-common v1.28.1/go.mod h1:Os8XeXGvHAap/p6M/8fQ3gle4eEXDGRQmoRNcPQTjXs= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= diff --git a/server/runtime_go.go b/server/runtime_go.go index 73a06c0501..13bab59bf4 100644 --- a/server/runtime_go.go +++ b/server/runtime_go.go @@ -2569,8 +2569,8 @@ func (ri *RuntimeGoInitializer) RegisterSubscriptionNotificationGoogle(fn func(c return nil } -func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int) error { - return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, maxEntries) +func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int, indexOnly bool) error { + return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, maxEntries, indexOnly) } func (ri *RuntimeGoInitializer) RegisterStorageIndexFilter(indexName string, fn func(ctx context.Context, logger runtime.Logger, db *sql.DB, nk runtime.NakamaModule, write *runtime.StorageWrite) bool) error { diff --git a/server/runtime_go_nakama.go b/server/runtime_go_nakama.go index 96ab635756..f0dce4f7bf 100644 --- a/server/runtime_go_nakama.go +++ b/server/runtime_go_nakama.go @@ -2051,7 +2051,7 @@ func (n *RuntimeGoNakamaModule) StorageDelete(ctx context.Context, deletes []*ru // @param limit(type=int) Maximum number of results to be returned. // @return objects(*api..StorageObjectList) A list of storage objects. // @return error(error) An optional error value if an error occurred. -func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error) { +func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error) { if indexName == "" { return nil, errors.New("expects a non-empty indexName") } @@ -2059,7 +2059,7 @@ func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, indexName, return nil, errors.New("limit must be 1-100") } - return n.storageIndex.List(ctx, indexName, query, limit, indexOnly) + return n.storageIndex.List(ctx, indexName, query, limit) } // @group users diff --git a/server/runtime_javascript_init.go b/server/runtime_javascript_init.go index c67b30bead..3dbacb11a2 100644 --- a/server/runtime_javascript_init.go +++ b/server/runtime_javascript_init.go @@ -1149,7 +1149,12 @@ func (im *RuntimeJavascriptInitModule) registerStorageIndex(r *goja.Runtime) fun idxMaxEntries := int(getJsInt(r, f.Argument(4))) - if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, idxMaxEntries); err != nil { + indexOnly := false + if !goja.IsUndefined(f.Argument(5)) && !goja.IsNull(f.Argument(5)) { + indexOnly = getJsBool(r, f.Argument(5)) + } + + if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, idxMaxEntries, indexOnly); err != nil { panic(r.NewGoError(fmt.Errorf("Failed to register storage index: %s", err.Error()))) } diff --git a/server/runtime_javascript_nakama.go b/server/runtime_javascript_nakama.go index 7448827b38..233ae271b1 100644 --- a/server/runtime_javascript_nakama.go +++ b/server/runtime_javascript_nakama.go @@ -358,12 +358,8 @@ func (n *runtimeJavascriptNakamaModule) storageIndexList(r *goja.Runtime) func(g panic(r.NewTypeError("limit must be 1-100")) } } - indexOnly := false - if !goja.IsUndefined(f.Argument(3)) && !goja.IsNull(f.Argument(3)) { - indexOnly = getJsBool(r, f.Argument(3)) - } - objectList, err := n.storageIndex.List(n.ctx, idxName, queryString, int(limit), indexOnly) + objectList, err := n.storageIndex.List(n.ctx, idxName, queryString, int(limit)) if err != nil { panic(r.NewGoError(fmt.Errorf("failed to lookup storage index: %s", err.Error()))) } diff --git a/server/runtime_lua_nakama.go b/server/runtime_lua_nakama.go index 272caf4e1a..66bbb16f35 100644 --- a/server/runtime_lua_nakama.go +++ b/server/runtime_lua_nakama.go @@ -523,8 +523,9 @@ func (n *RuntimeLuaNakamaModule) registerStorageIndex(l *lua.LState) int { fields = append(fields, v.String()) }) maxEntries := l.CheckInt(5) + indexOnly := l.OptBool(6, false) - if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, maxEntries); err != nil { + if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, maxEntries, indexOnly); err != nil { l.RaiseError("failed to create storage index: %s", err.Error()) } @@ -9843,9 +9844,8 @@ func (n *RuntimeLuaNakamaModule) storageIndexList(l *lua.LState) int { l.ArgError(3, "invalid limit: expects value 1-100") return 0 } - indexOnly := l.OptBool(4, false) - objectList, err := n.storageIndex.List(l.Context(), idxName, queryString, limit, indexOnly) + objectList, err := n.storageIndex.List(l.Context(), idxName, queryString, limit) if err != nil { l.RaiseError(err.Error()) return 0 diff --git a/server/storage_index.go b/server/storage_index.go index bcd7a709a1..bddd28901e 100644 --- a/server/storage_index.go +++ b/server/storage_index.go @@ -35,9 +35,9 @@ import ( type StorageIndex interface { Write(ctx context.Context, objects []*api.StorageObject) (creates int, deletes int) Delete(ctx context.Context, objects StorageOpDeletes) (deletes int) - List(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error) + List(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error) Load(ctx context.Context) error - CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int) error + CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int, indexOnly bool) error RegisterFilters(runtime *Runtime) } @@ -47,6 +47,7 @@ type storageIndex struct { Collection string Key string Fields []string + IndexOnly bool Index *bluge.Writer } @@ -117,7 +118,7 @@ func (si *LocalStorageIndex) Write(ctx context.Context, objects []*api.StorageOb } } - doc, err := si.mapIndexStorageFields(so.UserId, so.Collection, so.Key, so.Version, so.Value, so.PermissionRead, so.PermissionWrite, so.CreateTime.AsTime(), so.UpdateTime.AsTime(), idx.Fields) + doc, err := si.mapIndexStorageFields(so.UserId, so.Collection, so.Key, so.Version, so.Value, so.PermissionRead, so.PermissionWrite, so.CreateTime.AsTime(), so.UpdateTime.AsTime(), idx.Fields, idx.IndexOnly) if err != nil { si.logger.Error("Failed to map storage object values to index", zap.Error(err)) continue @@ -210,7 +211,7 @@ func (si *LocalStorageIndex) Delete(ctx context.Context, objects StorageOpDelete return deletes } -func (si *LocalStorageIndex) List(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error) { +func (si *LocalStorageIndex) List(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error) { idx, found := si.indexByName[indexName] if !found { return nil, fmt.Errorf("index %q not found", indexName) @@ -249,7 +250,7 @@ func (si *LocalStorageIndex) List(ctx context.Context, indexName, query string, return &api.StorageObjects{Objects: []*api.StorageObject{}}, nil } - if indexOnly { + if idx.IndexOnly { objects := make([]*api.StorageObject, 0, len(indexResults)) for _, idxResult := range indexResults { objects = append(objects, &api.StorageObject{ @@ -383,7 +384,7 @@ LIMIT $2` } } - doc, err := si.mapIndexStorageFields(dbUserID.String(), idx.Collection, dbKey, dbVersion, dbValue, dbRead, dbWrite, dbCreateTime, dbUpdateTime, idx.Fields) + doc, err := si.mapIndexStorageFields(dbUserID.String(), idx.Collection, dbKey, dbVersion, dbValue, dbRead, dbWrite, dbCreateTime, dbUpdateTime, idx.Fields, idx.IndexOnly) if err != nil { rows.Close() si.logger.Error("Failed to map storage object values to index", zap.Error(err)) @@ -433,7 +434,7 @@ LIMIT $2` return nil } -func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, version, value string, read, write int32, createTime, updateTime time.Time, filters []string) (*bluge.Document, error) { +func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, version, value string, read, write int32, createTime, updateTime time.Time, filters []string, indexOnly bool) (*bluge.Document, error) { if collection == "" || key == "" || userID == "" { return nil, errors.New("insufficient fields to create index document id") } @@ -467,11 +468,14 @@ func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, vers rv.AddField(bluge.NewKeywordField("version", version).StoreValue()) rv.AddField(bluge.NewNumericField("read", float64(read)).StoreValue()) rv.AddField(bluge.NewNumericField("write", float64(write)).StoreValue()) - json, err := json.Marshal(mapValue) - if err != nil { - return nil, err + + if indexOnly { + json, err := json.Marshal(mapValue) + if err != nil { + return nil, err + } + rv.AddField(bluge.NewStoredOnlyField("json", json)) } - rv.AddField(bluge.NewStoredOnlyField("json", json)) BlugeWalkDocument(mapValue, []string{"value"}, rv) @@ -482,7 +486,7 @@ type indexResult struct { Collection string Key string UserID string - Value string + Value string // Only returned for indices created with indexOnly == true Read int32 Write int32 Version string @@ -569,7 +573,7 @@ func (si *LocalStorageIndex) queryMatchesToDocumentIds(dmi search.DocumentMatchI return ids, nil } -func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int) error { +func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int, indexOnly bool) error { if name == "" { return errors.New("storage index 'name' must be set") } @@ -599,6 +603,7 @@ func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, Fields: fields, MaxEntries: maxEntries, Index: idx, + IndexOnly: indexOnly, } si.indexByName[name] = storageIdx @@ -618,6 +623,7 @@ func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, "key": cfgKey, "fields": fields, "max_entries": maxEntries, + "index_only": indexOnly, })) return nil diff --git a/server/storage_index_test.go b/server/storage_index_test.go index b3cdc7c84b..04c167319c 100644 --- a/server/storage_index_test.go +++ b/server/storage_index_test.go @@ -59,12 +59,12 @@ func TestLocalStorageIndex_Write(t *testing.T) { t.Fatal(err.Error()) } - if err := storageIdx.CreateIndex(ctx, indexName1, collection1, key, []string{"one", "two"}, maxEntries1); err != nil { + if err := storageIdx.CreateIndex(ctx, indexName1, collection1, key, []string{"one", "two"}, maxEntries1, false); err != nil { t.Fatal(err.Error()) } // Matches all keys - if err := storageIdx.CreateIndex(ctx, indexName2, collection1, "", []string{"three"}, maxEntries2); err != nil { + if err := storageIdx.CreateIndex(ctx, indexName2, collection1, "", []string{"three"}, maxEntries2, false); err != nil { t.Fatal(err.Error()) } @@ -124,13 +124,13 @@ func TestLocalStorageIndex_Write(t *testing.T) { t.Fatal(err.Error()) } - entries, err := storageIdx.List(ctx, indexName1, "", maxEntries1, false) // Match all + entries, err := storageIdx.List(ctx, indexName1, "", maxEntries1) // Match all if err != nil { t.Fatal(err.Error()) } assert.Len(t, entries.Objects, 2, "indexed results length was not 2") - entries, err = storageIdx.List(ctx, indexName2, "", maxEntries1, false) // Match all + entries, err = storageIdx.List(ctx, indexName2, "", maxEntries1) // Match all if err != nil { t.Fatal(err.Error()) } @@ -171,7 +171,7 @@ func TestLocalStorageIndex_Write(t *testing.T) { t.Fatal(err.Error()) } - entries, err := storageIdx.List(ctx, indexName1, "+value.three:3", maxEntries1, false) + entries, err := storageIdx.List(ctx, indexName1, "+value.three:3", maxEntries1) if err != nil { t.Fatal(err.Error()) } @@ -274,7 +274,7 @@ func TestLocalStorageIndex_Write(t *testing.T) { t.Fatal(err.Error()) } - entries, err := storageIdx.List(ctx, indexName2, "", maxEntries2, false) + entries, err := storageIdx.List(ctx, indexName2, "", maxEntries2) if err != nil { t.Fatal(err.Error()) } @@ -335,7 +335,7 @@ func TestLocalStorageIndex_List(t *testing.T) { t.Fatal(err.Error()) } - if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries); err != nil { + if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries, true); err != nil { t.Fatal(err.Error()) } @@ -370,7 +370,7 @@ func TestLocalStorageIndex_List(t *testing.T) { t.Fatal(err.Error()) } - entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10, false) + entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10) if err != nil { t.Fatal(err.Error()) } @@ -428,7 +428,7 @@ func TestLocalStorageIndex_List(t *testing.T) { t.Fatal(err.Error()) } - if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries); err != nil { + if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries, true); err != nil { t.Fatal(err.Error()) } @@ -463,7 +463,7 @@ func TestLocalStorageIndex_List(t *testing.T) { t.Fatal(err.Error()) } - entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10, true) + entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10) if err != nil { t.Fatal(err.Error()) } @@ -512,7 +512,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) { t.Fatal(err.Error()) } - if err := storageIdx.CreateIndex(ctx, indexName, collection, "", []string{"one"}, maxEntries); err != nil { + if err := storageIdx.CreateIndex(ctx, indexName, collection, "", []string{"one"}, maxEntries, false); err != nil { t.Fatal(err.Error()) } @@ -539,7 +539,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) { t.Fatal(err.Error()) } - entries, err := storageIdx.List(ctx, indexName, "", 10, false) + entries, err := storageIdx.List(ctx, indexName, "", 10) if err != nil { t.Fatal(err.Error()) } @@ -556,7 +556,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) { t.Fatal(err.Error()) } - entries, err = storageIdx.List(ctx, indexName, "", 10, false) + entries, err = storageIdx.List(ctx, indexName, "", 10) if err != nil { t.Fatal(err.Error()) } diff --git a/vendor/github.com/heroiclabs/nakama-common/runtime/runtime.go b/vendor/github.com/heroiclabs/nakama-common/runtime/runtime.go index ecad464753..acd2b02f8f 100644 --- a/vendor/github.com/heroiclabs/nakama-common/runtime/runtime.go +++ b/vendor/github.com/heroiclabs/nakama-common/runtime/runtime.go @@ -838,7 +838,7 @@ type Initializer interface { RegisterEventSessionEnd(fn func(ctx context.Context, logger Logger, evt *api.Event)) error // Register a new storage index. - RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int) error + RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int, indexOnly bool) error // RegisterStorageIndexFilter can be used to define a filtering function for a given storage index. RegisterStorageIndexFilter(indexName string, fn func(ctx context.Context, logger Logger, db *sql.DB, nk NakamaModule, write *StorageWrite) bool) error @@ -1070,7 +1070,7 @@ type NakamaModule interface { StorageRead(ctx context.Context, reads []*StorageRead) ([]*api.StorageObject, error) StorageWrite(ctx context.Context, writes []*StorageWrite) ([]*api.StorageObjectAck, error) StorageDelete(ctx context.Context, deletes []*StorageDelete) error - StorageIndexList(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error) + StorageIndexList(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error) MultiUpdate(ctx context.Context, accountUpdates []*AccountUpdate, storageWrites []*StorageWrite, walletUpdates []*WalletUpdate, updateLedger bool) ([]*api.StorageObjectAck, []*WalletUpdateResult, error)