Revert "Move indexOnly from index registration to listing"

This reverts commit b81de3a.

CHANGELOG.md

@@ -10,7 +10,7 @@ The format is based on [keep a changelog](http://keepachangelog.com) and this pr
 
 ### Changed
 - Add create_time and update_time to returned storage engine writes acks.
-- Add storageIndexList indexOnly param to read data only from the index.
+- Add storage index create flag to read only from the index.
 
 ### [3.17.1] - 2023-08-23
 ### Added

go.sum

@@ -292,6 +292,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/heroiclabs/nakama-common v1.28.1 h1:GOH8r27NBSdiBf7xjy2R/Ov7CSr9zp4DVJt9Y2LIbqg=
+github.com/heroiclabs/nakama-common v1.28.1/go.mod h1:Os8XeXGvHAap/p6M/8fQ3gle4eEXDGRQmoRNcPQTjXs=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=

server/runtime_go.go

@@ -2569,8 +2569,8 @@ func (ri *RuntimeGoInitializer) RegisterSubscriptionNotificationGoogle(fn func(c
 	return nil
 }
 
-func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int) error {
-	return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, maxEntries)
+func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int, indexOnly bool) error {
+	return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, maxEntries, indexOnly)
 }
 
 func (ri *RuntimeGoInitializer) RegisterStorageIndexFilter(indexName string, fn func(ctx context.Context, logger runtime.Logger, db *sql.DB, nk runtime.NakamaModule, write *runtime.StorageWrite) bool) error {

server/runtime_go_nakama.go

@@ -2051,15 +2051,15 @@ func (n *RuntimeGoNakamaModule) StorageDelete(ctx context.Context, deletes []*ru
 // @param limit(type=int) Maximum number of results to be returned.
 // @return objects(*api..StorageObjectList) A list of storage objects.
 // @return error(error) An optional error value if an error occurred.
-func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error) {
+func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error) {
 	if indexName == "" {
 		return nil, errors.New("expects a non-empty indexName")
 	}
 	if limit < 1 || limit > 100 {
 		return nil, errors.New("limit must be 1-100")
 	}
 
-	return n.storageIndex.List(ctx, indexName, query, limit, indexOnly)
+	return n.storageIndex.List(ctx, indexName, query, limit)
 }
 
 // @group users

server/runtime_javascript_init.go

@@ -1149,7 +1149,12 @@ func (im *RuntimeJavascriptInitModule) registerStorageIndex(r *goja.Runtime) fun
 
 		idxMaxEntries := int(getJsInt(r, f.Argument(4)))
 
-		if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, idxMaxEntries); err != nil {
+		indexOnly := false
+		if !goja.IsUndefined(f.Argument(5)) && !goja.IsNull(f.Argument(5)) {
+			indexOnly = getJsBool(r, f.Argument(5))
+		}
+
+		if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, idxMaxEntries, indexOnly); err != nil {
 			panic(r.NewGoError(fmt.Errorf("Failed to register storage index: %s", err.Error())))
 		}
 

server/runtime_javascript_nakama.go

@@ -358,12 +358,8 @@ func (n *runtimeJavascriptNakamaModule) storageIndexList(r *goja.Runtime) func(g
 				panic(r.NewTypeError("limit must be 1-100"))
 			}
 		}
-		indexOnly := false
-		if !goja.IsUndefined(f.Argument(3)) && !goja.IsNull(f.Argument(3)) {
-			indexOnly = getJsBool(r, f.Argument(3))
-		}
 
-		objectList, err := n.storageIndex.List(n.ctx, idxName, queryString, int(limit), indexOnly)
+		objectList, err := n.storageIndex.List(n.ctx, idxName, queryString, int(limit))
 		if err != nil {
 			panic(r.NewGoError(fmt.Errorf("failed to lookup storage index: %s", err.Error())))
 		}

server/runtime_lua_nakama.go

@@ -523,8 +523,9 @@ func (n *RuntimeLuaNakamaModule) registerStorageIndex(l *lua.LState) int {
 		fields = append(fields, v.String())
 	})
 	maxEntries := l.CheckInt(5)
+	indexOnly := l.OptBool(6, false)
 
-	if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, maxEntries); err != nil {
+	if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, maxEntries, indexOnly); err != nil {
 		l.RaiseError("failed to create storage index: %s", err.Error())
 	}
 
@@ -9843,9 +9844,8 @@ func (n *RuntimeLuaNakamaModule) storageIndexList(l *lua.LState) int {
 		l.ArgError(3, "invalid limit: expects value 1-100")
 		return 0
 	}
-	indexOnly := l.OptBool(4, false)
 
-	objectList, err := n.storageIndex.List(l.Context(), idxName, queryString, limit, indexOnly)
+	objectList, err := n.storageIndex.List(l.Context(), idxName, queryString, limit)
 	if err != nil {
 		l.RaiseError(err.Error())
 		return 0

server/storage_index.go

@@ -35,9 +35,9 @@ import (
 type StorageIndex interface {
 	Write(ctx context.Context, objects []*api.StorageObject) (creates int, deletes int)
 	Delete(ctx context.Context, objects StorageOpDeletes) (deletes int)
-	List(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error)
+	List(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error)
 	Load(ctx context.Context) error
-	CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int) error
+	CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int, indexOnly bool) error
 	RegisterFilters(runtime *Runtime)
 }
 
@@ -47,6 +47,7 @@ type storageIndex struct {
 	Collection string
 	Key        string
 	Fields     []string
+	IndexOnly  bool
 	Index      *bluge.Writer
 }
 
@@ -117,7 +118,7 @@ func (si *LocalStorageIndex) Write(ctx context.Context, objects []*api.StorageOb
 					}
 				}
 
-				doc, err := si.mapIndexStorageFields(so.UserId, so.Collection, so.Key, so.Version, so.Value, so.PermissionRead, so.PermissionWrite, so.CreateTime.AsTime(), so.UpdateTime.AsTime(), idx.Fields)
+				doc, err := si.mapIndexStorageFields(so.UserId, so.Collection, so.Key, so.Version, so.Value, so.PermissionRead, so.PermissionWrite, so.CreateTime.AsTime(), so.UpdateTime.AsTime(), idx.Fields, idx.IndexOnly)
 				if err != nil {
 					si.logger.Error("Failed to map storage object values to index", zap.Error(err))
 					continue
@@ -210,7 +211,7 @@ func (si *LocalStorageIndex) Delete(ctx context.Context, objects StorageOpDelete
 	return deletes
 }
 
-func (si *LocalStorageIndex) List(ctx context.Context, indexName, query string, limit int, indexOnly bool) (*api.StorageObjects, error) {
+func (si *LocalStorageIndex) List(ctx context.Context, indexName, query string, limit int) (*api.StorageObjects, error) {
 	idx, found := si.indexByName[indexName]
 	if !found {
 		return nil, fmt.Errorf("index %q not found", indexName)
@@ -249,7 +250,7 @@ func (si *LocalStorageIndex) List(ctx context.Context, indexName, query string,
 		return &api.StorageObjects{Objects: []*api.StorageObject{}}, nil
 	}
 
-	if indexOnly {
+	if idx.IndexOnly {
 		objects := make([]*api.StorageObject, 0, len(indexResults))
 		for _, idxResult := range indexResults {
 			objects = append(objects, &api.StorageObject{
@@ -383,7 +384,7 @@ LIMIT $2`
 				}
 			}
 
-			doc, err := si.mapIndexStorageFields(dbUserID.String(), idx.Collection, dbKey, dbVersion, dbValue, dbRead, dbWrite, dbCreateTime, dbUpdateTime, idx.Fields)
+			doc, err := si.mapIndexStorageFields(dbUserID.String(), idx.Collection, dbKey, dbVersion, dbValue, dbRead, dbWrite, dbCreateTime, dbUpdateTime, idx.Fields, idx.IndexOnly)
 			if err != nil {
 				rows.Close()
 				si.logger.Error("Failed to map storage object values to index", zap.Error(err))
@@ -433,7 +434,7 @@ LIMIT $2`
 	return nil
 }
 
-func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, version, value string, read, write int32, createTime, updateTime time.Time, filters []string) (*bluge.Document, error) {
+func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, version, value string, read, write int32, createTime, updateTime time.Time, filters []string, indexOnly bool) (*bluge.Document, error) {
 	if collection == "" || key == "" || userID == "" {
 		return nil, errors.New("insufficient fields to create index document id")
 	}
@@ -467,11 +468,14 @@ func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, vers
 	rv.AddField(bluge.NewKeywordField("version", version).StoreValue())
 	rv.AddField(bluge.NewNumericField("read", float64(read)).StoreValue())
 	rv.AddField(bluge.NewNumericField("write", float64(write)).StoreValue())
-	json, err := json.Marshal(mapValue)
-	if err != nil {
-		return nil, err
+
+	if indexOnly {
+		json, err := json.Marshal(mapValue)
+		if err != nil {
+			return nil, err
+		}
+		rv.AddField(bluge.NewStoredOnlyField("json", json))
 	}
-	rv.AddField(bluge.NewStoredOnlyField("json", json))
 
 	BlugeWalkDocument(mapValue, []string{"value"}, rv)
 
@@ -482,7 +486,7 @@ type indexResult struct {
 	Collection string
 	Key        string
 	UserID     string
-	Value      string
+	Value      string // Only returned for indices created with indexOnly == true
 	Read       int32
 	Write      int32
 	Version    string
@@ -569,7 +573,7 @@ func (si *LocalStorageIndex) queryMatchesToDocumentIds(dmi search.DocumentMatchI
 	return ids, nil
 }
 
-func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int) error {
+func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int, indexOnly bool) error {
 	if name == "" {
 		return errors.New("storage index 'name' must be set")
 	}
@@ -599,6 +603,7 @@ func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection,
 		Fields:     fields,
 		MaxEntries: maxEntries,
 		Index:      idx,
+		IndexOnly:  indexOnly,
 	}
 	si.indexByName[name] = storageIdx
 
@@ -618,6 +623,7 @@ func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection,
 		"key":         cfgKey,
 		"fields":      fields,
 		"max_entries": maxEntries,
+		"index_only":  indexOnly,
 	}))
 
 	return nil

server/storage_index_test.go

@@ -59,12 +59,12 @@ func TestLocalStorageIndex_Write(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if err := storageIdx.CreateIndex(ctx, indexName1, collection1, key, []string{"one", "two"}, maxEntries1); err != nil {
+	if err := storageIdx.CreateIndex(ctx, indexName1, collection1, key, []string{"one", "two"}, maxEntries1, false); err != nil {
 		t.Fatal(err.Error())
 	}
 
 	// Matches all keys
-	if err := storageIdx.CreateIndex(ctx, indexName2, collection1, "", []string{"three"}, maxEntries2); err != nil {
+	if err := storageIdx.CreateIndex(ctx, indexName2, collection1, "", []string{"three"}, maxEntries2, false); err != nil {
 		t.Fatal(err.Error())
 	}
 
@@ -124,13 +124,13 @@ func TestLocalStorageIndex_Write(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		entries, err := storageIdx.List(ctx, indexName1, "", maxEntries1, false) // Match all
+		entries, err := storageIdx.List(ctx, indexName1, "", maxEntries1) // Match all
 		if err != nil {
 			t.Fatal(err.Error())
 		}
 		assert.Len(t, entries.Objects, 2, "indexed results length was not 2")
 
-		entries, err = storageIdx.List(ctx, indexName2, "", maxEntries1, false) // Match all
+		entries, err = storageIdx.List(ctx, indexName2, "", maxEntries1) // Match all
 		if err != nil {
 			t.Fatal(err.Error())
 		}
@@ -171,7 +171,7 @@ func TestLocalStorageIndex_Write(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		entries, err := storageIdx.List(ctx, indexName1, "+value.three:3", maxEntries1, false)
+		entries, err := storageIdx.List(ctx, indexName1, "+value.three:3", maxEntries1)
 		if err != nil {
 			t.Fatal(err.Error())
 		}
@@ -274,7 +274,7 @@ func TestLocalStorageIndex_Write(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		entries, err := storageIdx.List(ctx, indexName2, "", maxEntries2, false)
+		entries, err := storageIdx.List(ctx, indexName2, "", maxEntries2)
 		if err != nil {
 			t.Fatal(err.Error())
 		}
@@ -335,7 +335,7 @@ func TestLocalStorageIndex_List(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries); err != nil {
+		if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries, true); err != nil {
 			t.Fatal(err.Error())
 		}
 
@@ -370,7 +370,7 @@ func TestLocalStorageIndex_List(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10, false)
+		entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10)
 		if err != nil {
 			t.Fatal(err.Error())
 		}
@@ -428,7 +428,7 @@ func TestLocalStorageIndex_List(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries); err != nil {
+		if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries, true); err != nil {
 			t.Fatal(err.Error())
 		}
 
@@ -463,7 +463,7 @@ func TestLocalStorageIndex_List(t *testing.T) {
 			t.Fatal(err.Error())
 		}
 
-		entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10, true)
+		entries, err := storageIdx.List(ctx, indexName, "value.one:1 value.three:3", 10)
 		if err != nil {
 			t.Fatal(err.Error())
 		}
@@ -512,7 +512,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if err := storageIdx.CreateIndex(ctx, indexName, collection, "", []string{"one"}, maxEntries); err != nil {
+	if err := storageIdx.CreateIndex(ctx, indexName, collection, "", []string{"one"}, maxEntries, false); err != nil {
 		t.Fatal(err.Error())
 	}
 
@@ -539,7 +539,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	entries, err := storageIdx.List(ctx, indexName, "", 10, false)
+	entries, err := storageIdx.List(ctx, indexName, "", 10)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -556,7 +556,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	entries, err = storageIdx.List(ctx, indexName, "", 10, false)
+	entries, err = storageIdx.List(ctx, indexName, "", 10)
 	if err != nil {
 		t.Fatal(err.Error())
 	}