diff --git a/.air.toml b/.air.toml index a372a2c28..8b59cb1a6 100644 --- a/.air.toml +++ b/.air.toml @@ -10,7 +10,7 @@ tmp_dir = "tmp" exclude_regex = [] exclude_unchanged = false follow_symlink = false - full_bin = "GO_ENV=development ./dicedb --enable-multithreading=false --enable-profiling=false" + full_bin = "GO_ENV=development ./dicedb --enable-multithreading=true --enable-profiling=false" include_dir = [] include_ext = ["go", "tpl", "tmpl", "yaml", "env"] kill_delay = "1s" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 4e06d1e43..36571ac22 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,15 +1,15 @@ -name: Linter +name: linter on: push: branches: [master] paths: - - '*.go' + - '**/*.go' pull_request: branches: [master] paths: - - '*.go' + - '**/*.go' permissions: contents: read @@ -28,4 +28,3 @@ jobs: uses: golangci/golangci-lint-action@v6 with: version: v1.60.1 - \ No newline at end of file diff --git a/docs/src/content/docs/commands/COPY.md b/docs/src/content/docs/commands/COPY.md index 144e754b7..e8edebfeb 100644 --- a/docs/src/content/docs/commands/COPY.md +++ b/docs/src/content/docs/commands/COPY.md @@ -17,7 +17,6 @@ COPY [DB destination-db] [REPLACE] | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | :------: | | source | The key of the value you want to copy. This key must exist. | String | Yes | | destination | The key where the value will be copied to. This key must not exist unless the `REPLACE` option is specified. | String | Yes | -| DB destination-db | The database number where the destination key will be created. If not specified, the destination key will be created in the same database as the source key. | Integer | optional | | REPLACE | If specified, the command will overwrite the destination key if it already exists. | String | Optional | ## Return Value diff --git a/go.mod b/go.mod index 9791cfecd..64bc75043 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,9 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -28,16 +28,16 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.11.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/arch v0.12.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.20.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( github.com/axiomhq/hyperloglog v0.2.0 - github.com/bytedance/sonic v1.12.3 + github.com/bytedance/sonic v1.12.4 github.com/cespare/xxhash/v2 v2.3.0 github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da @@ -57,7 +57,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.8 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 - golang.org/x/crypto v0.28.0 - golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c + golang.org/x/crypto v0.29.0 + golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f google.golang.org/protobuf v1.35.1 ) diff --git a/go.sum b/go.sum index 2b7c9f001..242b25ad3 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= +github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -36,6 +38,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -52,6 +56,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -121,18 +127,28 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= +golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/integration_tests/commands/async/discard_test.go b/integration_tests/commands/async/discard_test.go deleted file mode 100644 index b26ef5c0a..000000000 --- a/integration_tests/commands/async/discard_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package async - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDiscard(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - testCases := []struct { - name string - cmds []string - expect []interface{} - }{ - { - name: "Discard commands in a txn", - cmds: []string{"MULTI", "SET key1 value1", "DISCARD"}, - expect: []interface{}{"OK", "QUEUED", "OK"}, - }, - { - name: "Throw error if Discard used outside a txn", - cmds: []string{"DISCARD"}, - expect: []interface{}{"ERR DISCARD without MULTI"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - //deleteTestKeys([]string{"key1"}, store) - FireCommand(conn, "DEL key1") - for i, cmd := range tc.cmds { - result := FireCommand(conn, cmd) - assert.Equal(t, tc.expect[i], result, "Value mismatch for cmd %s", cmd) - } - }) - } -} diff --git a/integration_tests/commands/async/json_test.go b/integration_tests/commands/async/json_test.go deleted file mode 100644 index 9bec702d0..000000000 --- a/integration_tests/commands/async/json_test.go +++ /dev/null @@ -1,1262 +0,0 @@ -package async - -import ( - "fmt" - "net" - "strings" - "testing" - - "github.com/bytedance/sonic" - "github.com/dicedb/dice/testutils" - "github.com/stretchr/testify/assert" -) - -func TestJSONOperations(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - simpleJSON := `{"name":"John","age":30}` - nestedJSON := `{"name":"Alice","address":{"city":"New York","zip":"10001"},"array":[1,2,3,4,5]}` - specialCharsJSON := `{"key":"value with spaces","emoji":"πŸ˜€"}` - unicodeJSON := `{"unicode":"γ“γ‚“γ«γ‘γ―δΈ–η•Œ"}` - escapedCharsJSON := `{"escaped":"\"quoted\", \\backslash\\ and /forward/slash"}` - complexJSON := `{"inventory":{"mountain_bikes":[{"id":"bike:1","model":"Phoebe","price":1920,"specs":{"material":"carbon","weight":13.1},"colors":["black","silver"]},{"id":"bike:2","model":"Quaoar","price":2072,"specs":{"material":"aluminium","weight":7.9},"colors":["black","white"]},{"id":"bike:3","model":"Weywot","price":3264,"specs":{"material":"alloy","weight":13.8}}],"commuter_bikes":[{"id":"bike:4","model":"Salacia","price":1475,"specs":{"material":"aluminium","weight":16.6},"colors":["black","silver"]},{"id":"bike:5","model":"Mimas","price":3941,"specs":{"material":"alloy","weight":11.6}}]}}` - - // Background: - // Ordering in JSON objects is not guaranteed - // Ordering in JSON arrays is guaranteed - // Ordering of arrays which are constructed using a key of an object is not maintained across objects - - // Single ordered test cases will cover all JSON operations which have a single possible order of expected elements - // different JSON Key orderings are not considered as different JSONs, hence will be considered in this category - // What goes here: - // - Cases where the possible order of the result is a single permutation - // - JSON Arrays fetched without any JSONPath - // - JSON Objects (key ordering is taken care of) - // ref: https://github.com/DiceDB/dice/pull/365 - singleOrderedTestCases := []struct { - name string - setCmd string - getCmd string - expected string - }{ - { - name: "Set and Get Integer", - setCmd: `JSON.SET tools $ 2`, - getCmd: `JSON.GET tools`, - expected: "2", - }, - { - name: "Set and Get Boolean True", - setCmd: `JSON.SET booleanTrue $ true`, - getCmd: `JSON.GET booleanTrue`, - expected: "true", - }, - { - name: "Set and Get Boolean False", - setCmd: `JSON.SET booleanFalse $ false`, - getCmd: `JSON.GET booleanFalse`, - expected: "false", - }, - { - name: "Set and Get Simple JSON", - setCmd: `JSON.SET user $ ` + simpleJSON, - getCmd: `JSON.GET user`, - expected: simpleJSON, - }, - { - name: "Set and Get Nested JSON", - setCmd: `JSON.SET user:2 $ ` + nestedJSON, - getCmd: `JSON.GET user:2`, - expected: nestedJSON, - }, - { - name: "Set and Get JSON Array", - setCmd: `JSON.SET numbers $ [1,2,3,4,5]`, - getCmd: `JSON.GET numbers`, - expected: `[1,2,3,4,5]`, - }, - { - name: "Set and Get JSON with Special Characters", - setCmd: `JSON.SET special $ ` + specialCharsJSON, - getCmd: `JSON.GET special`, - expected: specialCharsJSON, - }, - { - name: "Get JSON with Wrong Number of Arguments", - setCmd: ``, - getCmd: `JSON.GET`, - expected: "ERR wrong number of arguments for 'json.get' command", - }, - { - name: "Set Non-JSON Value", - setCmd: `SET nonJson "not a json"`, - getCmd: `JSON.GET nonJson`, - expected: "ERR Existing key has wrong Dice type", - }, - { - name: "Set Empty JSON Object", - setCmd: `JSON.SET empty $ {}`, - getCmd: `JSON.GET empty`, - expected: `{}`, - }, - { - name: "Set Empty JSON Array", - setCmd: `JSON.SET emptyArray $ []`, - getCmd: `JSON.GET emptyArray`, - expected: `[]`, - }, - { - name: "Set JSON with Unicode", - setCmd: `JSON.SET unicode $ ` + unicodeJSON, - getCmd: `JSON.GET unicode`, - expected: unicodeJSON, - }, - { - name: "Set JSON with Escaped Characters", - setCmd: `JSON.SET escaped $ ` + escapedCharsJSON, - getCmd: `JSON.GET escaped`, - expected: escapedCharsJSON, - }, - { - name: "Set and Get Complex JSON", - setCmd: `JSON.SET inventory $ ` + complexJSON, - getCmd: `JSON.GET inventory`, - expected: complexJSON, - }, - { - name: "Get Nested Array", - setCmd: `JSON.SET inventory $ ` + complexJSON, - getCmd: `JSON.GET inventory $.inventory.mountain_bikes[*].model`, - expected: `["Phoebe","Quaoar","Weywot"]`, - }, - { - name: "Get Nested Object", - setCmd: `JSON.SET inventory $ ` + complexJSON, - getCmd: `JSON.GET inventory $.inventory.mountain_bikes[0].specs`, - expected: `{"material":"carbon","weight":13.1}`, - }, - { - name: "Set Nested Value", - setCmd: `JSON.SET inventory $.inventory.mountain_bikes[0].price 2000`, - getCmd: `JSON.GET inventory $.inventory.mountain_bikes[0].price`, - expected: `2000`, - }, - { - name: "Get JSON with non-existent path", - setCmd: `JSON.SET user $ ` + simpleJSON, - getCmd: `JSON.GET user $.nonExistent`, - expected: `ERR Path '$.nonExistent' does not exist`, - }, - } - - // Multiple test cases will address JSON operations where the order of elements can vary, but all orders are "valid" and to be accepted - // The variation in order is due to the inherent nature of JSON objects. - // When dealing with a resultant array that contains elements from multiple objects, the order of these elements can have several valid permutations. - // Which then means, the overall order of elements in the resultant array is not fixed, although each sub-array within it is guaranteed to be ordered. - // What goes here: - // - Cases where the possible order of the resultant array is multiple permutations - // ref: https://github.com/DiceDB/dice/pull/365 - multipleOrderedTestCases := []struct { - name string - setCmd string - getCmd string - expected []string - }{ - { - name: "Get All Prices", - setCmd: `JSON.SET inventory $ ` + complexJSON, - getCmd: `JSON.GET inventory $..price`, - expected: []string{`[1475,3941,1920,2072,3264]`, `[1920,2072,3264,1475,3941]`}, // Ordering agnostic - }, - { - name: "Set Multiple Nested Values", - setCmd: `JSON.SET inventory $.inventory.*[?(@.price<2000)].price 1500`, - getCmd: `JSON.GET inventory $..price`, - expected: []string{`[1500,3941,1500,2072,3264]`, `[1500,2072,3264,1500,3941]`}, // Ordering agnostic - }, - } - - t.Run("Single Ordered Test Cases", func(t *testing.T) { - for _, tc := range singleOrderedTestCases { - t.Run(tc.name, func(t *testing.T) { - if tc.setCmd != "" { - result := FireCommand(conn, tc.setCmd) - assert.Equal(t, "OK", result) - } - - if tc.getCmd != "" { - result := FireCommand(conn, tc.getCmd) - if testutils.IsJSONResponse(result.(string)) { - assert.JSONEq(t, tc.expected, result.(string)) - } else { - assert.Equal(t, tc.expected, result) - } - } - }) - } - }) - - t.Run("Multiple Ordered Test Cases", func(t *testing.T) { - for _, tc := range multipleOrderedTestCases { - t.Run(tc.name, func(t *testing.T) { - if tc.setCmd != "" { - result := FireCommand(conn, tc.setCmd) - assert.Equal(t, "OK", result) - } - - if tc.getCmd != "" { - result := FireCommand(conn, tc.getCmd) - testutils.AssertJSONEqualList(t, tc.expected, result.(string)) - } - }) - } - }) -} - -func TestJSONSetWithInvalidJSON(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - testCases := []struct { - name string - command string - expected string - }{ - { - name: "Set Invalid JSON", - command: `JSON.SET invalid $ {invalid:json}`, - expected: "ERR invalid JSON", - }, - { - name: "Set JSON with Wrong Number of Arguments", - command: `JSON.SET`, - expected: "ERR wrong number of arguments for 'json.set' command", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := FireCommand(conn, tc.command) - assert.True(t, strings.HasPrefix(result.(string), tc.expected), fmt.Sprintf("Expected: %s, Got: %s", tc.expected, result)) - }) - } -} - -func TestUnsupportedJSONPathPatterns(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - complexJSON := `{"inventory":{"mountain_bikes":[{"id":"bike:1","model":"Phoebe","price":1920,"specs":{"material":"carbon","weight":13.1},"colors":["black","silver"]},{"id":"bike:2","model":"Quaoar","price":2072,"specs":{"material":"aluminium","weight":7.9},"colors":["black","white"]},{"id":"bike:3","model":"Weywot","price":3264,"specs":{"material":"alloy","weight":13.8}}],"commuter_bikes":[{"id":"bike:4","model":"Salacia","price":1475,"specs":{"material":"aluminium","weight":16.6},"colors":["black","silver"]},{"id":"bike:5","model":"Mimas","price":3941,"specs":{"material":"alloy","weight":11.6}}]}}` - - setupCmd := `JSON.SET bikes:inventory $ ` + complexJSON - result := FireCommand(conn, setupCmd) - assert.Equal(t, "OK", result) - - testCases := []struct { - name string - command string - expected string - }{ - { - name: "Regex in JSONPath", - command: `JSON.GET bikes:inventory '$..[?(@.specs.material =~ "(?i)al")].model'`, - expected: "ERR invalid JSONPath", - }, - { - name: "Using @ for referencing other fields", - command: `JSON.GET bikes:inventory '$.inventory.mountain_bikes[?(@.specs.material =~ @.regex_pat)].model'`, - expected: "ERR invalid JSONPath", - }, - { - name: "Complex condition with multiple comparisons", - command: `JSON.GET bikes:inventory '$..mountain_bikes[?(@.price < 3000 && @.specs.weight < 10)]'`, - expected: "ERR invalid JSONPath", - }, - { - name: "Get all colors", - command: `JSON.GET bikes:inventory '$..[*].colors'`, - expected: "ERR invalid JSONPath", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := FireCommand(conn, tc.command) - assert.Equal(t, tc.expected, result) - }) - } -} - -func TestJSONSetWithNXAndXX(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - FireCommand(conn, "DEL user") - - user1 := `{"name":"John","age":30}` - user2 := `{"name":"Rahul","age":28}` - - testCases := []struct { - name string - commands []string - expected []interface{} - }{ - { - name: "Set with XX on non-existent key", - commands: []string{"JSON.SET user $ " + user1 + " XX", "JSON.SET user $ " + user1 + " NX", "JSON.GET user"}, - expected: []interface{}{"(nil)", "OK", user1}, - }, - { - name: "Set with NX on existing key", - commands: []string{"DEL user", "JSON.SET user $ " + user1, "JSON.SET user $ " + user1 + " NX"}, - expected: []interface{}{int64(1), "OK", "(nil)"}, - }, - { - name: "Set with XX on existing key", - commands: []string{"JSON.SET user $ " + user2 + " XX", "JSON.GET user", "DEL user"}, - expected: []interface{}{"OK", user2, int64(1)}, - }, - { - name: "Set with NX on non-existent key", - commands: []string{"JSON.SET user $ " + user2 + " NX", "JSON.SET user $ " + user1 + " NX", "JSON.GET user"}, - expected: []interface{}{"OK", "(nil)", user2}, - }, - { - name: "Invalid combinations of NX and XX", - commands: []string{"JSON.SET user $ " + user2 + " NX NX", "JSON.SET user $ " + user2 + " NX XX", "JSON.SET user $ " + user2 + " NX Abcd", "JSON.SET user $ " + user2 + " NX "}, - expected: []interface{}{"ERR syntax error", "ERR syntax error", "ERR syntax error", "(nil)"}, - }, - { - name: "Invalid combinations of XX", - commands: []string{"JSON.SET user $ " + user2 + " XX XX", "JSON.SET user $ " + user2 + " XX NX", "JSON.SET user $ " + user2 + " XX Abcd", "JSON.SET user $ " + user2 + " XX "}, - expected: []interface{}{"ERR syntax error", "ERR syntax error", "ERR syntax error", "OK"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for i, cmd := range tc.commands { - result := FireCommand(conn, cmd) - jsonResult, isString := result.(string) - if isString && testutils.IsJSONResponse(jsonResult) { - assert.JSONEq(t, tc.expected[i].(string), jsonResult) - } else { - assert.Equal(t, tc.expected[i], result) - } - } - }) - } -} - -func TestJSONClearOperations(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - // deleteTestKeys([]string{"user"}, store) - FireCommand(conn, "DEL user") - - testCases := []struct { - name string - commands []string - expected []interface{} - }{ - { - name: "jsonclear root path", - commands: []string{ - `JSON.SET user $ {"age":13,"high":1.60,"flag":true,"name":"jerry","pet":null,"language":["python","golang"],"partner":{"name":"tom","language":["rust"]}}`, - "JSON.CLEAR user $", - "JSON.GET user $", - }, - expected: []interface{}{"OK", int64(1), "{}"}, - }, - { - name: "jsonclear string type", - commands: []string{ - `JSON.SET user $ {"name":"Tom","age":30}`, - "JSON.CLEAR user $.name", - "JSON.GET user $.name", - }, - expected: []interface{}{"OK", int64(0), `"Tom"`}, - }, - { - name: "jsonclear array type", - commands: []string{ - `JSON.SET user $ {"names":["Rahul","Tom"],"ages":[25,30]}`, - "JSON.CLEAR user $.names", - "JSON.GET user $.names", - }, - expected: []interface{}{"OK", int64(1), "[]"}, - }, - { - name: "jsonclear bool type", - commands: []string{ - `JSON.SET user $ {"flag":true,"name":"Tom"}`, - "JSON.CLEAR user $.flag", - "JSON.GET user $.flag"}, - expected: []interface{}{"OK", int64(0), "true"}, - }, - { - name: "jsonclear null type", - commands: []string{ - `JSON.SET user $ {"name":null,"age":28}`, - "JSON.CLEAR user $.pet", - "JSON.GET user $.name"}, - expected: []interface{}{"OK", int64(0), "null"}, - }, - { - name: "jsonclear integer type", - commands: []string{ - `JSON.SET user $ {"age":28,"name":"Tom"}`, - "JSON.CLEAR user $.age", - "JSON.GET user $.age"}, - expected: []interface{}{"OK", int64(1), "0"}, - }, - { - name: "jsonclear float type", - commands: []string{ - `JSON.SET user $ {"price":3.14,"name":"sugar"}`, - "JSON.CLEAR user $.price", - "JSON.GET user $.price"}, - expected: []interface{}{"OK", int64(1), "0"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for i, cmd := range tc.commands { - result := FireCommand(conn, cmd) - assert.Equal(t, tc.expected[i], result) - } - }) - } -} - -func TestJSONDelOperations(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - FireCommand(conn, "DEL user") - - testCases := []struct { - name string - commands []string - expected []interface{} - }{ - { - name: "Delete root path", - commands: []string{ - `JSON.SET user $ {"age":13,"high":1.60,"flag":true,"name":"jerry","pet":null,"language":["python","golang"],"partner":{"name":"tom","language":["rust"]}}`, - "JSON.DEL user $", - "JSON.GET user $", - }, - expected: []interface{}{"OK", int64(1), "(nil)"}, - }, - { - name: "Delete nested field", - commands: []string{ - `JSON.SET user $ {"name":"Tom","address":{"city":"New York","zip":"10001"}}`, - "JSON.DEL user $.address.city", - "JSON.GET user $", - }, - expected: []interface{}{"OK", int64(1), `{"name":"Tom","address":{"zip":"10001"}}`}, - }, - { - name: "del string type", - commands: []string{ - `JSON.SET user $ {"flag":true,"name":"Tom"}`, - "JSON.DEL user $.name", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"flag":true}`}, - }, - { - name: "del bool type", - commands: []string{ - `JSON.SET user $ {"flag":true,"name":"Tom"}`, - "JSON.DEL user $.flag", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"name":"Tom"}`}, - }, - { - name: "del null type", - commands: []string{ - `JSON.SET user $ {"name":null,"age":28}`, - "JSON.DEL user $.name", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"age":28}`}, - }, - { - name: "del array type", - commands: []string{ - `JSON.SET user $ {"names":["Rahul","Tom"],"bosses":{"names":["Jerry","Rocky"],"hobby":"swim"}}`, - "JSON.DEL user $..names", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(2), `{"bosses":{"hobby":"swim"}}`}, - }, - { - name: "del integer type", - commands: []string{ - `JSON.SET user $ {"age":28,"name":"Tom"}`, - "JSON.DEL user $.age", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"name":"Tom"}`}, - }, - { - name: "del float type", - commands: []string{ - `JSON.SET user $ {"price":3.14,"name":"sugar"}`, - "JSON.DEL user $.price", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"name":"sugar"}`}, - }, - { - name: "delete key with []", - commands: []string{ - `JSON.SET data $ {"key[0]":"value","array":["a","b"]}`, - `JSON.DEL data ["key[0]"]`, - "JSON.GET data $"}, - expected: []interface{}{"OK", int64(1), `{"array": ["a","b"]}`}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for i, cmd := range tc.commands { - result := FireCommand(conn, cmd) - stringResult, ok := result.(string) - if ok && testutils.IsJSONResponse(stringResult) { - assert.JSONEq(t, tc.expected[i].(string), stringResult) - } else { - assert.Equal(t, tc.expected[i], result) - } - } - }) - } -} - -func TestJSONForgetOperations(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - FireCommand(conn, "FORGET user") - - testCases := []struct { - name string - commands []string - expected []interface{} - }{ - { - name: "Forget root path", - commands: []string{ - `JSON.SET user $ {"age":13,"high":1.60,"flag":true,"name":"jerry","pet":null,"language":["python","golang"],"partner":{"name":"tom","language":["rust"]}}`, - "JSON.FORGET user $", - "JSON.GET user $", - }, - expected: []interface{}{"OK", int64(1), "(nil)"}, - }, - { - name: "Forget nested field", - commands: []string{ - `JSON.SET user $ {"name":"Tom","address":{"city":"New York","zip":"10001"}}`, - "JSON.FORGET user $.address.city", - "JSON.GET user $", - }, - expected: []interface{}{"OK", int64(1), `{"name":"Tom","address":{"zip":"10001"}}`}, - }, - { - name: "forget string type", - commands: []string{`JSON.SET user $ {"flag":true,"name":"Tom"}`, - "JSON.FORGET user $.name", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"flag":true}`}, - }, - { - name: "forget bool type", - commands: []string{`JSON.SET user $ {"flag":true,"name":"Tom"}`, - "JSON.FORGET user $.flag", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"name":"Tom"}`}, - }, - { - name: "forget null type", - commands: []string{`JSON.SET user $ {"name":null,"age":28}`, - "JSON.FORGET user $.name", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"age":28}`}, - }, - { - name: "forget array type", - commands: []string{`JSON.SET user $ {"names":["Rahul","Tom"],"bosses":{"names":["Jerry","Rocky"],"hobby":"swim"}}`, - "JSON.FORGET user $..names", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(2), `{"bosses":{"hobby":"swim"}}`}, - }, - { - name: "forget integer type", - commands: []string{`JSON.SET user $ {"age":28,"name":"Tom"}`, - "JSON.FORGET user $.age", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"name":"Tom"}`}, - }, - { - name: "forget float type", - commands: []string{`JSON.SET user $ {"price":3.14,"name":"sugar"}`, - "JSON.FORGET user $.price", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"name":"sugar"}`}, - }, - { - name: "forget array element", - commands: []string{`JSON.SET user $ {"names":["Rahul","Tom"],"bosses":{"names":["Jerry","Rocky"],"hobby":"swim"}}`, - "JSON.FORGET user $.names[0]", - "JSON.GET user $"}, - expected: []interface{}{"OK", int64(1), `{"names":["Tom"],"bosses":{"names":["Jerry","Rocky"],"hobby":"swim"}}`}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for i, cmd := range tc.commands { - result := FireCommand(conn, cmd) - stringResult, ok := result.(string) - if ok && testutils.IsJSONResponse(stringResult) { - assert.JSONEq(t, tc.expected[i].(string), stringResult) - } else { - assert.Equal(t, tc.expected[i], result) - } - } - }) - } -} - -func arraysArePermutations[T comparable](a, b []T) bool { - // If lengths are different, they cannot be permutations - if len(a) != len(b) { - return false - } - - // Count occurrences of each element in array 'a' - countA := make(map[T]int) - for _, elem := range a { - countA[elem]++ - } - - // Subtract occurrences based on array 'b' - for _, elem := range b { - countA[elem]-- - if countA[elem] < 0 { - return false - } - } - - // Check if all counts are zero - for _, count := range countA { - if count != 0 { - return false - } - } - - return true -} - -func TestJsonStrlen(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - FireCommand(conn, "DEL doc") - - testCases := []struct { - name string - commands []string - expected []interface{} - }{ - { - name: "jsonstrlen with root path", - commands: []string{ - `JSON.SET doc $ ["hello","world"]`, - "JSON.STRLEN doc $", - }, - expected: []interface{}{"OK", []interface{}{"(nil)"}}, - }, - { - name: "jsonstrlen nested", - commands: []string{ - `JSON.SET doc $ {"name":"jerry","partner":{"name":"tom"}}`, - "JSON.STRLEN doc $..name", - }, - expected: []interface{}{"OK", []interface{}{int64(5), int64(3)}}, - }, - { - name: "jsonstrlen with no path and object at root", - commands: []string{ - `JSON.SET doc $ {"name":"bhima","age":10}`, - "JSON.STRLEN doc", - }, - expected: []interface{}{"OK", "ERR wrong type of path value - expected string but found object"}, - }, - { - name: "jsonstrlen with no path and object at boolean", - commands: []string{ - `JSON.SET doc $ true`, - "JSON.STRLEN doc", - }, - expected: []interface{}{"OK", "ERR wrong type of path value - expected string but found boolean"}, - }, - { - name: "jsonstrlen with no path and object at array", - commands: []string{ - `JSON.SET doc $ [1,2,3,4]`, - "JSON.STRLEN doc", - }, - expected: []interface{}{"OK", "ERR wrong type of path value - expected string but found array"}, - }, - { - name: "jsonstrlen with no path and object at integer", - commands: []string{ - `JSON.SET doc $ 1`, - "JSON.STRLEN doc", - }, - expected: []interface{}{"OK", "ERR wrong type of path value - expected string but found integer"}, - }, - { - name: "jsonstrlen with no path and object at number", - commands: []string{ - `JSON.SET doc $ 1.9`, - "JSON.STRLEN doc", - }, - expected: []interface{}{"OK", "ERR wrong type of path value - expected string but found number"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for i, cmd := range tc.commands { - result := FireCommand(conn, cmd) - stringResult, ok := result.(string) - if ok { - assert.Equal(t, tc.expected[i], stringResult) - } else { - assert.True(t, arraysArePermutations(tc.expected[i].([]interface{}), result.([]interface{}))) - } - } - }) - } -} - -func TestJSONMGET(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - defer FireCommand(conn, "DEL xx yy zz p q r t u v doc1 doc2") - - setupData := map[string]string{ - "xx": `["hehhhe","hello"]`, - "yy": `{"name":"jerry","partner":{"name":"jerry","language":["rust"]},"partner2":{"language":["rust"]}}`, - "zz": `{"name":"tom","partner":{"name":"tom","language":["rust"]},"partner2":{"age":12,"language":["rust"]}}`, - "doc1": `{"a":1,"b":2,"nested":{"a":3},"c":null}`, - "doc2": `{"a":4,"b":5,"nested":{"a":6},"c":null}`, - } - - for key, value := range setupData { - resp := FireCommand(conn, fmt.Sprintf("JSON.SET %s $ %s", key, value)) - assert.Equal(t, "OK", resp) - } - - testCases := []struct { - name string - command string - expected []interface{} - }{ - { - name: "MGET with root path", - command: "JSON.MGET xx yy zz $", - expected: []interface{}{setupData["xx"], setupData["yy"], setupData["zz"]}, - }, - { - name: "MGET with specific path", - command: "JSON.MGET xx yy zz $.name", - expected: []interface{}{"(nil)", `"jerry"`, `"tom"`}, - }, - { - name: "MGET with nested path", - command: "JSON.MGET xx yy zz $.partner2.age", - expected: []interface{}{"(nil)", "(nil)", "12"}, - }, - { - name: "MGET error", - command: "JSON.MGET t", - expected: []interface{}{"ERR wrong number of arguments for 'json.mget' command"}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := FireCommand(conn, tc.command) - results, ok := result.([]interface{}) - if ok { - assert.Equal(t, len(tc.expected), len(results)) - for i := range results { - if testutils.IsJSONResponse(tc.expected[i].(string)) { - assert.JSONEq(t, tc.expected[i].(string), results[i].(string)) - } else { - assert.Equal(t, tc.expected[i], results[i]) - } - } - } else { - assert.Equal(t, tc.expected[0], result) - } - }) - } - - t.Run("MGET with recursive path", testJSONMGETRecursive(conn)) -} - -func testJSONMGETRecursive(conn net.Conn) func(*testing.T) { - return func(t *testing.T) { - result := FireCommand(conn, "JSON.MGET doc1 doc2 $..a") - results, ok := result.([]interface{}) - assert.True(t, ok, "Expected result to be a slice of interface{}") - assert.Equal(t, 2, len(results), "Expected 2 results") - - expectedSets := [][]int{ - {1, 3}, - {4, 6}, - } - - for i, res := range results { - var actualSet []int - err := sonic.UnmarshalString(res.(string), &actualSet) - assert.Nil(t, err, "Failed to unmarshal JSON") - - assert.True(t, len(actualSet) == len(expectedSets[i]), - "Mismatch in number of elements for set %d", i) - - for _, expected := range expectedSets[i] { - assert.True(t, sliceContainsItem(actualSet, expected), - "Set %d does not contain expected value %d", i, expected) - } - } - } -} - -// sliceContainsItem checks if a slice sliceContainsItem a given item -func sliceContainsItem(slice []int, item int) bool { - for _, v := range slice { - if v == item { - return true - } - } - return false -} - -func TestJsonARRAPPEND(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - a := `[1,2]` - b := `{"name":"jerry","partner":{"name":"tom","score":[10]},"partner2":{"score":[10,20]}}` - c := `{"name":["jerry"],"partner":{"name":"tom","score":[10]},"partner2":{"name":12,"score":"rust"}}` - - testCases := []struct { - name string - commands []string - expected []interface{} - assertType []string - }{ - - { - name: "JSON.ARRAPPEND with root path", - commands: []string{"json.set a $ " + a, `json.arrappend a $ 3`}, - expected: []interface{}{"OK", []interface{}{int64(3)}}, - assertType: []string{"equal", "deep_equal"}, - }, - { - name: "JSON.ARRAPPEND nested", - commands: []string{"JSON.SET doc $ " + b, `JSON.ARRAPPEND doc $..score 10`}, - expected: []interface{}{"OK", []interface{}{int64(2), int64(3)}}, - assertType: []string{"equal", "deep_equal"}, - }, - { - name: "JSON.ARRAPPEND nested with nil", - commands: []string{"JSON.SET doc $ " + c, `JSON.ARRAPPEND doc $..score 10`}, - expected: []interface{}{"OK", []interface{}{int64(2), "(nil)"}}, - assertType: []string{"equal", "deep_equal"}, - }, - { - name: "JSON.ARRAPPEND with different datatypes", - commands: []string{"JSON.SET doc $ " + c, "JSON.ARRAPPEND doc $.name 1"}, - expected: []interface{}{"OK", []interface{}{int64(2)}}, - assertType: []string{"equal", "deep_equal"}, - }, - } - for _, tcase := range testCases { - FireCommand(conn, "DEL a") - FireCommand(conn, "DEL doc") - t.Run(tcase.name, func(t *testing.T) { - for i := 0; i < len(tcase.commands); i++ { - cmd := tcase.commands[i] - out := tcase.expected[i] - result := FireCommand(conn, cmd) - if tcase.assertType[i] == "equal" { - assert.Equal(t, out, result) - } else if tcase.assertType[i] == "deep_equal" { - assert.True(t, arraysArePermutations(out.([]interface{}), result.([]interface{}))) - } - } - }) - } -} - -func deStringify(input string) []string { - input = strings.Replace(input, "[", "", -1) - input = strings.Replace(input, "]", "", -1) - arrayString := strings.Split(input, ",") - for i, v := range arrayString { - arrayString[i] = strings.TrimSpace(v) - } - return arrayString -} - -func TestJsonNummultby(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - a := `{"a":"b","b":[{"a":2},{"a":5},{"a":"c"}]}` - invalidArgMessage := "ERR wrong number of arguments for 'json.nummultby' command" - - testCases := []struct { - name string - commands []string - expected []interface{} - assertType []string - }{ - { - name: "Invalid number of arguments", - commands: []string{"JSON.NUMMULTBY ", "JSON.NUMMULTBY docu", "JSON.NUMMULTBY docu $"}, - expected: []interface{}{invalidArgMessage, invalidArgMessage, invalidArgMessage}, - assertType: []string{"equal", "equal", "equal"}, - }, - { - name: "MultBy at non-existent key", - commands: []string{"JSON.NUMMULTBY docu $ 1"}, - expected: []interface{}{"ERR could not perform this operation on a key that doesn't exist"}, - assertType: []string{"equal"}, - }, - { - name: "Invalid value of multiplier on non-existent key", - commands: []string{"JSON.SET docu $ " + a, "JSON.NUMMULTBY docu $.fe x"}, - expected: []interface{}{"OK", "[]"}, - assertType: []string{"equal", "equal"}, - }, - { - name: "Invalid value of multiplier on existent key", - commands: []string{"JSON.SET docu $ " + a, "JSON.NUMMULTBY docu $.a x"}, - expected: []interface{}{"OK", "ERR expected value at line 1 column 1"}, - assertType: []string{"equal", "equal"}, - }, - { - name: "MultBy at recursive path", - commands: []string{"JSON.SET docu $ " + a, "JSON.NUMMULTBY docu $..a 2"}, - expected: []interface{}{"OK", "[4,10,null,null]"}, - assertType: []string{"equal", "deep_equal"}, - }, - { - name: "MultBy at root path", - commands: []string{"JSON.SET docu $ " + a, "JSON.NUMMULTBY docu $.a 2"}, - expected: []interface{}{"OK", "[null]"}, - assertType: []string{"equal", "deep_equal"}, - }, - } - - for _, tcase := range testCases { - FireCommand(conn, "DEL docu") - t.Run(tcase.name, func(t *testing.T) { - for i := 0; i < len(tcase.commands); i++ { - cmd := tcase.commands[i] - out := tcase.expected[i] - result := FireCommand(conn, cmd) - if tcase.assertType[i] == "equal" { - assert.Equal(t, out, result) - } else if tcase.assertType[i] == "deep_equal" { - assert.True(t, arraysArePermutations(deStringify(out.(string)), deStringify(result.(string)))) - } - } - }) - } -} - -func TestJsonObjLen(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - a := `{"name":"jerry","partner":{"name":"tom","language":["rust"]}}` - b := `{"name":"jerry","partner":{"name":"tom","language":["rust"]},"partner2":{"name":"spike","language":["go","rust"]}}` - c := `{"name":"jerry","partner":{"name":"tom","language":["rust"]},"partner2":{"name":12,"language":["rust"]}}` - d := `["this","is","an","array"]` - - defer func() { - resp := FireCommand(conn, "DEL obj") - assert.Equal(t, int64(1), resp) - }() - - testCases := []struct { - name string - commands []string - expected []interface{} - }{ - { - name: "JSON.OBJLEN with root path", - commands: []string{"json.set obj $ " + a, "json.objlen obj $"}, - expected: []interface{}{"OK", []interface{}{int64(2)}}, - }, - { - name: "JSON.OBJLEN with nested path", - commands: []string{"json.set obj $ " + b, "json.objlen obj $.partner"}, - expected: []interface{}{"OK", []interface{}{int64(2)}}, - }, - { - name: "JSON.OBJLEN with non-object path", - commands: []string{"json.set obj $ " + d, "json.objlen obj $"}, - expected: []interface{}{"OK", []interface{}{"(nil)"}}, - }, - { - name: "JSON.OBJLEN with nested non-object path", - commands: []string{"json.set obj $ " + c, "json.objlen obj $.partner2.name"}, - expected: []interface{}{"OK", []interface{}{"(nil)"}}, - }, - { - name: "JSON.OBJLEN nested objects", - commands: []string{"json.set obj $ " + b, "json.objlen obj $..language"}, - expected: []interface{}{"OK", []interface{}{"(nil)", "(nil)"}}, - }, - { - name: "JSON.OBJLEN invalid json path", - commands: []string{"json.set obj $ " + b, "json.objlen obj $..language*something"}, - expected: []interface{}{"OK", "ERR Path '$..language*something' does not exist"}, - }, - { - name: "JSON.OBJLEN with non-existent key", - commands: []string{"json.set obj $ " + b, "json.objlen non_existing_key $"}, - expected: []interface{}{"OK", "(nil)"}, - }, - { - name: "JSON.OBJLEN with empty path", - commands: []string{"json.set obj $ " + a, "json.objlen obj"}, - expected: []interface{}{"OK", int64(2)}, - }, - { - name: "JSON.OBJLEN invalid json path2", - commands: []string{"json.set obj $ " + c, "json.objlen obj $[1"}, - expected: []interface{}{"OK", "ERR Path '$[1' does not exist"}, - }, - { - name: "JSON.OBJLEN invalid json path", - commands: []string{"json.set obj $ " + c, "json.objlen"}, - expected: []interface{}{"OK", "ERR wrong number of arguments for 'json.objlen' command"}, - }, - { - name: "JSON.OBJLEN with legacy path - root", - commands: []string{"json.set obj $ " + c, "json.objlen obj ."}, - expected: []interface{}{"OK", int64(3)}, - }, - { - name: "JSON.OBJLEN with legacy path - inner existing path", - commands: []string{"json.set obj $ " + c, "json.objlen obj .partner", "json.objlen obj .partner2"}, - expected: []interface{}{"OK", int64(2), int64(2)}, - }, - { - name: "JSON.OBJLEN with legacy path - inner existing path v2", - commands: []string{"json.set obj $ " + c, "json.objlen obj partner", "json.objlen obj partner2"}, - expected: []interface{}{"OK", int64(2), int64(2)}, - }, - { - name: "JSON.OBJLEN with legacy path - inner non-existent path", - commands: []string{"json.set obj $ " + c, "json.objlen obj .idonotexist"}, - expected: []interface{}{"OK", "(nil)"}, - }, - { - name: "JSON.OBJLEN with legacy path - inner non-existent path v2", - commands: []string{"json.set obj $ " + c, "json.objlen obj idonotexist"}, - expected: []interface{}{"OK", "(nil)"}, - }, - { - name: "JSON.OBJLEN with legacy path - inner existent path with nonJSON object", - commands: []string{"json.set obj $ " + c, "json.objlen obj .name"}, - expected: []interface{}{"OK", "WRONGTYPE Operation against a key holding the wrong kind of value"}, - }, - { - name: "JSON.OBJLEN with legacy path - inner existent path recursive object", - commands: []string{"json.set obj $ " + c, "json.objlen obj ..partner"}, - expected: []interface{}{"OK", int64(2)}, - }, - } - - for _, tcase := range testCases { - FireCommand(conn, "DEL obj") - t.Run(tcase.name, func(t *testing.T) { - for i := 0; i < len(tcase.commands); i++ { - cmd := tcase.commands[i] - out := tcase.expected[i] - result := FireCommand(conn, cmd) - - assert.Equal(t, out, result) - } - }) - } -} - -func convertToArray(input string) []string { - input = strings.Trim(input, `"[`) - input = strings.Trim(input, `]"`) - elements := strings.Split(input, ",") - for i, element := range elements { - elements[i] = strings.TrimSpace(element) - } - return elements -} - -func TestJSONNumIncrBy(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - invalidArgMessage := "ERR wrong number of arguments for 'json.numincrby' command" - testCases := []struct { - name string - setupData string - commands []string - expected []interface{} - assertType []string - cleanUp []string - }{ - { - name: "Invalid number of arguments", - setupData: "", - commands: []string{"JSON.NUMINCRBY ", "JSON.NUMINCRBY foo", "JSON.NUMINCRBY foo $"}, - expected: []interface{}{invalidArgMessage, invalidArgMessage, invalidArgMessage}, - assertType: []string{"equal", "equal", "equal"}, - cleanUp: []string{}, - }, - { - name: "Non-existent key", - setupData: "", - commands: []string{"JSON.NUMINCRBY foo $ 1"}, - expected: []interface{}{"ERR could not perform this operation on a key that doesn't exist"}, - assertType: []string{"equal"}, - cleanUp: []string{}, - }, - { - name: "Invalid value of increment", - setupData: "JSON.SET foo $ 1", - commands: []string{"JSON.GET foo $", "JSON.NUMINCRBY foo $ @", "JSON.NUMINCRBY foo $ 122@"}, - expected: []interface{}{"1", "ERR expected value at line 1 column 1", "ERR trailing characters at line 1 column 4"}, - assertType: []string{"equal", "equal", "equal"}, - cleanUp: []string{"DEL foo"}, - }, - { - name: "incrby at non root path", - setupData: fmt.Sprintf("JSON.SET %s $ %s", "foo", `{"a":"b","b":[{"a":2.2},{"a":5},{"a":"c"}]}`), - commands: []string{"JSON.NUMINCRBY foo $..a 2", "JSON.NUMINCRBY foo $.a 2", "JSON.GET foo", "JSON.NUMINCRBY foo $..a -2", "JSON.GET foo"}, - expected: []interface{}{"[null,4.2,7,null]", "[null]", "{\"a\":\"b\",\"b\":[{\"a\":4.2},{\"a\":7},{\"a\":\"c\"}]}", "[null,2.2,5,null]", "{\"a\":\"b\",\"b\":[{\"a\":2.2},{\"a\":5},{\"a\":\"c\"}]}"}, - assertType: []string{"perm_equal", "perm_equal", "json_equal", "perm_equal", "json_equal"}, - cleanUp: []string{"DEL foo"}, - }, - { - name: "incrby at root path", - setupData: "JSON.SET foo $ 1", - commands: []string{"JSON.NUMINCRBY foo $ 1", "JSON.GET foo $", "JSON.NUMINCRBY foo $ -1", "JSON.GET foo $"}, - expected: []interface{}{"[2]", "2", "[1]", "1"}, - assertType: []string{"equal", "equal", "equal", "equal"}, - cleanUp: []string{"DEL foo"}, - }, - { - name: "incrby at root path", - setupData: "JSON.SET foo $ 1", - commands: []string{"expire foo 10", "JSON.NUMINCRBY foo $ 1", "ttl foo", "JSON.GET foo $", "JSON.NUMINCRBY foo $ -1", "JSON.GET foo $"}, - expected: []interface{}{int64(1), "[2]", int64(10), "2", "[1]", "1"}, - assertType: []string{"equal", "equal", "range", "equal", "equal", "equal"}, - cleanUp: []string{"DEL foo"}, - }, - } - - for _, tc := range testCases { - FireCommand(conn, "DEL foo") - t.Run(tc.name, func(t *testing.T) { - if tc.setupData != "" { - assert.Equal(t, FireCommand(conn, tc.setupData), "OK") - } - for i := 0; i < len(tc.commands); i++ { - cmd := tc.commands[i] - out := tc.expected[i] - result := FireCommand(conn, cmd) - switch tc.assertType[i] { - case "equal": - assert.Equal(t, out, result) - case "perm_equal": - assert.True(t, arraysArePermutations(convertToArray(out.(string)), convertToArray(result.(string)))) - case "range": - assert.True(t, result.(int64) <= tc.expected[i].(int64) && result.(int64) > 0, "Expected %v to be within 0 to %v", result, tc.expected[i]) - case "json_equal": - assert.JSONEq(t, out.(string), result.(string)) - } - } - for i := 0; i < len(tc.cleanUp); i++ { - FireCommand(conn, tc.cleanUp[i]) - } - }) - } -} - -func TestJsonSTRAPPEND(t *testing.T) { - conn := getLocalConnection() - defer conn.Close() - - simpleJSON := `{"name":"John","age":30}` - - testCases := []struct { - name string - setCmd string - getCmd string - expected interface{} - }{ - { - name: "STRAPPEND to nested string", - setCmd: `JSON.SET doc $ ` + simpleJSON, - getCmd: `JSON.STRAPPEND doc $.name " Doe"`, - expected: []interface{}{int64(8)}, - }, - { - name: "STRAPPEND to multiple paths", - setCmd: `JSON.SET doc $ ` + simpleJSON, - getCmd: `JSON.STRAPPEND doc $..name "baz"`, - expected: []interface{}{int64(7)}, - }, - { - name: "STRAPPEND to non-string", - setCmd: `JSON.SET doc $ ` + simpleJSON, - getCmd: `JSON.STRAPPEND doc $.age " years"`, - expected: []interface{}{"(nil)"}, - }, - { - name: "STRAPPEND with empty string", - setCmd: `JSON.SET doc $ ` + simpleJSON, - getCmd: `JSON.STRAPPEND doc $.name ""`, - expected: []interface{}{int64(4)}, - }, - { - name: "STRAPPEND to non-existent path", - setCmd: `JSON.SET doc $ ` + simpleJSON, - getCmd: `JSON.STRAPPEND doc $.nonexistent " test"`, - expected: []interface{}{}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Use FLUSHDB to clear all keys before each test - result := FireCommand(conn, "FLUSHDB") - assert.Equal(t, "OK", result) - - result = FireCommand(conn, tc.setCmd) - assert.Equal(t, "OK", result) - - result = FireCommand(conn, tc.getCmd) - assert.ElementsMatch(t, tc.expected, result) - - }) - } -} diff --git a/integration_tests/commands/async/jsonresp_test.go b/integration_tests/commands/async/jsonresp_test.go index 82d72c216..f72f17a82 100644 --- a/integration_tests/commands/async/jsonresp_test.go +++ b/integration_tests/commands/async/jsonresp_test.go @@ -6,6 +6,36 @@ import ( "github.com/stretchr/testify/assert" ) +func arraysArePermutations[T comparable](a, b []T) bool { + // If lengths are different, they cannot be permutations + if len(a) != len(b) { + return false + } + + // Count occurrences of each element in array 'a' + countA := make(map[T]int) + for _, elem := range a { + countA[elem]++ + } + + // Subtract occurrences based on array 'b' + for _, elem := range b { + countA[elem]-- + if countA[elem] < 0 { + return false + } + } + + // Check if all counts are zero + for _, count := range countA { + if count != 0 { + return false + } + } + + return true +} + func TestJSONRESP(t *testing.T) { conn := getLocalConnection() defer conn.Close() diff --git a/integration_tests/commands/async/object_test.go b/integration_tests/commands/async/object_test.go index f434264c4..6ede26830 100644 --- a/integration_tests/commands/async/object_test.go +++ b/integration_tests/commands/async/object_test.go @@ -71,11 +71,11 @@ func TestObjectCommand(t *testing.T) { }, { name: "Object Encoding check for json", - commands: []string{`JSON.SET k1 $ ` + simpleJSON, "OBJECT ENCODING k1"}, + commands: []string{`JSON.SET k10 $ ` + simpleJSON, "OBJECT ENCODING k10"}, expected: []interface{}{"OK", "json"}, assertType: []string{"equal", "equal"}, delay: []time.Duration{0, 0}, - cleanup: []string{"DEL k1"}, + cleanup: []string{"DEL k10"}, }, { name: "Object Encoding check for bytearray", diff --git a/integration_tests/commands/http/deque_test.go b/integration_tests/commands/http/deque_test.go index 96f48031f..db315a559 100644 --- a/integration_tests/commands/http/deque_test.go +++ b/integration_tests/commands/http/deque_test.go @@ -12,11 +12,6 @@ import ( var deqRandGenerator *rand.Rand var deqRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_!@#$%^&*()-=+[]\\;':,.<>/?~.|") -var ( - deqNormalValues []string - deqEdgeValues []string -) - func deqRandStr(n int) string { b := make([]rune, n) for i := range b { @@ -25,7 +20,7 @@ func deqRandStr(n int) string { return string(b) } -func deqTestInit() { +func deqTestInit() (deqNormalValues, deqEdgeValues []string) { randSeed := time.Now().UnixNano() deqRandGenerator = rand.New(rand.NewSource(randSeed)) fmt.Printf("rand seed: %v", randSeed) @@ -66,10 +61,11 @@ func deqTestInit() { "-9223372036854775808", // min 64 bit int "9223372036854775807", // max 64 bit int } + return deqNormalValues, deqEdgeValues } func TestLPush(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -112,7 +108,7 @@ func TestLPush(t *testing.T) { } func TestRPush(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -155,7 +151,7 @@ func TestRPush(t *testing.T) { } func TestLPushLPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -233,7 +229,7 @@ func TestLPushLPop(t *testing.T) { } func TestLPushRPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -312,7 +308,7 @@ func TestLPushRPop(t *testing.T) { } func TestRPushLPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -391,7 +387,7 @@ func TestRPushLPop(t *testing.T) { } func TestRPushRPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -470,7 +466,6 @@ func TestRPushRPop(t *testing.T) { } func TestLRPushLRPop(t *testing.T) { - deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -511,7 +506,6 @@ func TestLRPushLRPop(t *testing.T) { } func TestLLEN(t *testing.T) { - deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) @@ -558,7 +552,6 @@ func TestLLEN(t *testing.T) { } func TestLPOPCount(t *testing.T) { - deqTestInit() exec := NewHTTPCommandExecutor() exec.FireCommand(HTTPCommand{Command: "FLUSHDB"}) diff --git a/integration_tests/commands/http/json_test.go b/integration_tests/commands/http/json_test.go index 195979ae1..f337613ef 100644 --- a/integration_tests/commands/http/json_test.go +++ b/integration_tests/commands/http/json_test.go @@ -157,7 +157,7 @@ func TestJSONOperations(t *testing.T) { {Command: "SET", Body: map[string]interface{}{"key": "k1", "value": "1"}}, {Command: "JSON.GET", Body: map[string]interface{}{"key": "k1"}}, }, - expected: []interface{}{"OK", "ERR Existing key has wrong Dice type"}, + expected: []interface{}{"OK", "WRONGTYPE Operation against a key holding the wrong kind of value"}, }, { name: "Set Empty JSON Object", diff --git a/integration_tests/commands/async/copy_test.go b/integration_tests/commands/resp/copy_test.go similarity index 80% rename from integration_tests/commands/async/copy_test.go rename to integration_tests/commands/resp/copy_test.go index 3fad1db47..95f8817ae 100644 --- a/integration_tests/commands/async/copy_test.go +++ b/integration_tests/commands/resp/copy_test.go @@ -1,7 +1,8 @@ -package async +package resp import ( "testing" + "time" "github.com/dicedb/dice/testutils" "github.com/stretchr/testify/assert" @@ -17,51 +18,61 @@ func TestCopy(t *testing.T) { name string commands []string expected []interface{} + delays []time.Duration }{ { name: "COPY when source key doesn't exist", commands: []string{"COPY k1 k2"}, expected: []interface{}{int64(0)}, + delays: []time.Duration{0}, }, { name: "COPY with no REPLACE", commands: []string{"SET k1 v1", "COPY k1 k2", "GET k1", "GET k2"}, expected: []interface{}{"OK", int64(1), "v1", "v1"}, + delays: []time.Duration{0, 0, 0, 0}, }, { name: "COPY with REPLACE", commands: []string{"SET k1 v1", "SET k2 v2", "GET k2", "COPY k1 k2 REPLACE", "GET k2"}, expected: []interface{}{"OK", "OK", "v2", int64(1), "v1"}, + delays: []time.Duration{0, 0, 0, 0, 0}, }, { name: "COPY with JSON integer", commands: []string{"JSON.SET k1 $ 2", "COPY k1 k2", "JSON.GET k2"}, expected: []interface{}{"OK", int64(1), "2"}, + delays: []time.Duration{0, 0, 0}, }, { name: "COPY with JSON boolean", commands: []string{"JSON.SET k1 $ true", "COPY k1 k2", "JSON.GET k2"}, expected: []interface{}{"OK", int64(1), "true"}, + delays: []time.Duration{0, 0, 0}, }, { name: "COPY with JSON array", commands: []string{`JSON.SET k1 $ [1,2,3]`, "COPY k1 k2", "JSON.GET k2"}, expected: []interface{}{"OK", int64(1), `[1,2,3]`}, + delays: []time.Duration{0, 0, 0}, }, { name: "COPY with JSON simple JSON", commands: []string{`JSON.SET k1 $ ` + simpleJSON, "COPY k1 k2", "JSON.GET k2"}, expected: []interface{}{"OK", int64(1), simpleJSON}, + delays: []time.Duration{0, 0, 0}, }, { name: "COPY with no expiry", commands: []string{"SET k1 v1", "COPY k1 k2", "TTL k1", "TTL k2"}, expected: []interface{}{"OK", int64(1), int64(-1), int64(-1)}, + delays: []time.Duration{0, 0, 0, 0}, }, { name: "COPY with expiry making sure copy expires", - commands: []string{"SET k1 v1 EX 5", "COPY k1 k2", "GET k1", "GET k2", "SLEEP 7", "GET k1", "GET k2"}, - expected: []interface{}{"OK", int64(1), "v1", "v1", "OK", "(nil)", "(nil)"}, + commands: []string{"SET k1 v1 EX 3", "COPY k1 k2", "GET k1", "GET k2", "GET k1", "GET k2"}, + expected: []interface{}{"OK", int64(1), "v1", "v1", "(nil)", "(nil)"}, + delays: []time.Duration{0, 0, 0, 0, 3 * time.Second, 0}, }, } @@ -75,6 +86,9 @@ func TestCopy(t *testing.T) { FireCommand(conn, "DEL k1") FireCommand(conn, "DEL k2") for i, cmd := range tc.commands { + if tc.delays[i] > 0 { + time.Sleep(tc.delays[i]) + } result := FireCommand(conn, cmd) resStr, resOk := result.(string) expStr, expOk := tc.expected[i].(string) diff --git a/integration_tests/commands/resp/deque_test.go b/integration_tests/commands/resp/deque_test.go index f9d13ea3d..82268398c 100644 --- a/integration_tests/commands/resp/deque_test.go +++ b/integration_tests/commands/resp/deque_test.go @@ -27,7 +27,7 @@ func deqRandStr(n int) string { return string(b) } -func deqTestInit() { +func deqTestInit() (deqNormalValues, deqEdgeValues []string) { randSeed := time.Now().UnixNano() deqRandGenerator = rand.New(rand.NewSource(randSeed)) fmt.Printf("rand seed: %v", randSeed) @@ -68,10 +68,11 @@ func deqTestInit() { "-9223372036854775808", // min 64 bit int "9223372036854775807", // max 64 bit int } + return deqNormalValues, deqEdgeValues } func TestLPush(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -110,7 +111,7 @@ func TestLPush(t *testing.T) { } func TestRPush(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -149,7 +150,7 @@ func TestRPush(t *testing.T) { } func TestLPushLPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -203,7 +204,7 @@ func TestLPushLPop(t *testing.T) { } func TestLPushRPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -257,7 +258,7 @@ func TestLPushRPop(t *testing.T) { } func TestRPushLPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -311,7 +312,7 @@ func TestRPushLPop(t *testing.T) { } func TestRPushRPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -365,7 +366,6 @@ func TestRPushRPop(t *testing.T) { } func TestLRPushLRPop(t *testing.T) { - deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -404,7 +404,6 @@ func TestLRPushLRPop(t *testing.T) { } func TestLLEN(t *testing.T) { - deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -531,7 +530,6 @@ func TestLRange(t *testing.T) { } func TestLPOPCount(t *testing.T) { - deqTestInit() conn := getLocalConnection() defer conn.Close() @@ -543,28 +541,28 @@ func TestLPOPCount(t *testing.T) { { name: "LPOP with count argument - valid, invalid, and edge cases", cmds: []string{ - "RPUSH k v1 v2 v3 v4", - "LPOP k 2", - "LLEN k", - "LPOP k 0", - "LLEN k", - "LPOP k 5", - "LLEN k", - "LPOP k -1", - "LPOP k abc", - "LLEN k", + "RPUSH k v1 v2 v3 v4", + "LPOP k 2", + "LLEN k", + "LPOP k 0", + "LLEN k", + "LPOP k 5", + "LLEN k", + "LPOP k -1", + "LPOP k abc", + "LLEN k", }, expect: []any{ - int64(4), - []interface{}{"v1", "v2"}, - int64(2), - "(nil)", - int64(2), - []interface{}{"v3", "v4"}, - int64(0), - "ERR value is not an integer or out of range", - "ERR value is not an integer or a float", - int64(0), + int64(4), + []interface{}{"v1", "v2"}, + int64(2), + "(nil)", + int64(2), + []interface{}{"v3", "v4"}, + int64(0), + "ERR value is not an integer or out of range", + "ERR value is not an integer or a float", + int64(0), }, }, } diff --git a/integration_tests/commands/async/dump_test.go b/integration_tests/commands/resp/dump_test.go similarity index 98% rename from integration_tests/commands/async/dump_test.go rename to integration_tests/commands/resp/dump_test.go index 7ec050b91..7e0f04fe8 100644 --- a/integration_tests/commands/async/dump_test.go +++ b/integration_tests/commands/resp/dump_test.go @@ -1,4 +1,4 @@ -package async +package resp import ( "encoding/base64" @@ -81,7 +81,7 @@ func TestDumpRestore(t *testing.T) { "DUMP nonexistentkey", }, expected: []interface{}{ - "ERR nil", + "(nil)", }, }, } diff --git a/integration_tests/commands/async/hgetall_test.go b/integration_tests/commands/resp/hgetall_test.go similarity index 96% rename from integration_tests/commands/async/hgetall_test.go rename to integration_tests/commands/resp/hgetall_test.go index 7971f0cc4..c179d9b7e 100644 --- a/integration_tests/commands/async/hgetall_test.go +++ b/integration_tests/commands/resp/hgetall_test.go @@ -1,4 +1,4 @@ -package async +package resp import ( "reflect" @@ -7,10 +7,6 @@ import ( "github.com/stretchr/testify/assert" ) -var ZERO int64 = 0 -var ONE int64 = 1 -var TWO int64 = 2 - func TestHGETALL(t *testing.T) { conn := getLocalConnection() defer conn.Close() diff --git a/integration_tests/commands/resp/json_test.go b/integration_tests/commands/resp/json_test.go index a2a9e9c70..c2c498a32 100644 --- a/integration_tests/commands/resp/json_test.go +++ b/integration_tests/commands/resp/json_test.go @@ -4,8 +4,10 @@ import ( "fmt" "net" "sort" + "strings" "testing" + "github.com/bytedance/sonic" "github.com/dicedb/dice/testutils" "github.com/stretchr/testify/assert" @@ -65,12 +67,351 @@ func runIntegrationTests(t *testing.T, conn net.Conn, testCases []IntegrationTes } } +func TestJSONOperations(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + + simpleJSON := `{"name":"John","age":30}` + nestedJSON := `{"name":"Alice","address":{"city":"New York","zip":"10001"},"array":[1,2,3,4,5]}` + specialCharsJSON := `{"key":"value with spaces","emoji":"πŸ˜€"}` + unicodeJSON := `{"unicode":"γ“γ‚“γ«γ‘γ―δΈ–η•Œ"}` + escapedCharsJSON := `{"escaped":"\"quoted\", \\backslash\\ and /forward/slash"}` + complexJSON := `{"inventory":{"mountain_bikes":[{"id":"bike:1","model":"Phoebe","price":1920,"specs":{"material":"carbon","weight":13.1},"colors":["black","silver"]},{"id":"bike:2","model":"Quaoar","price":2072,"specs":{"material":"aluminium","weight":7.9},"colors":["black","white"]},{"id":"bike:3","model":"Weywot","price":3264,"specs":{"material":"alloy","weight":13.8}}],"commuter_bikes":[{"id":"bike:4","model":"Salacia","price":1475,"specs":{"material":"aluminium","weight":16.6},"colors":["black","silver"]},{"id":"bike:5","model":"Mimas","price":3941,"specs":{"material":"alloy","weight":11.6}}]}}` + + // Background: + // Ordering in JSON objects is not guaranteed + // Ordering in JSON arrays is guaranteed + // Ordering of arrays which are constructed using a key of an object is not maintained across objects + + // Single ordered test cases will cover all JSON operations which have a single possible order of expected elements + // different JSON Key orderings are not considered as different JSONs, hence will be considered in this category + // What goes here: + // - Cases where the possible order of the result is a single permutation + // - JSON Arrays fetched without any JSONPath + // - JSON Objects (key ordering is taken care of) + // ref: https://github.com/DiceDB/dice/pull/365 + singleOrderedTestCases := []struct { + name string + setCmd string + getCmd string + expected string + }{ + { + name: "Set and Get Integer", + setCmd: `JSON.SET tools $ 2`, + getCmd: `JSON.GET tools`, + expected: "2", + }, + { + name: "Set and Get Boolean True", + setCmd: `JSON.SET booleanTrue $ true`, + getCmd: `JSON.GET booleanTrue`, + expected: "true", + }, + { + name: "Set and Get Boolean False", + setCmd: `JSON.SET booleanFalse $ false`, + getCmd: `JSON.GET booleanFalse`, + expected: "false", + }, + { + name: "Set and Get Simple JSON", + setCmd: `JSON.SET user $ ` + simpleJSON, + getCmd: `JSON.GET user`, + expected: simpleJSON, + }, + { + name: "Set and Get Nested JSON", + setCmd: `JSON.SET user:2 $ ` + nestedJSON, + getCmd: `JSON.GET user:2`, + expected: nestedJSON, + }, + { + name: "Set and Get JSON Array", + setCmd: `JSON.SET numbers $ [1,2,3,4,5]`, + getCmd: `JSON.GET numbers`, + expected: `[1,2,3,4,5]`, + }, + { + name: "Set and Get JSON with Special Characters", + setCmd: `JSON.SET special $ ` + specialCharsJSON, + getCmd: `JSON.GET special`, + expected: specialCharsJSON, + }, + { + name: "Get JSON with Wrong Number of Arguments", + setCmd: ``, + getCmd: `JSON.GET`, + expected: "ERR wrong number of arguments for 'json.get' command", + }, + { + name: "Set Non-JSON Value", + setCmd: `SET nonJson "not a json"`, + getCmd: `JSON.GET nonJson`, + expected: "WRONGTYPE Operation against a key holding the wrong kind of value", + }, + { + name: "Set Empty JSON Object", + setCmd: `JSON.SET empty $ {}`, + getCmd: `JSON.GET empty`, + expected: `{}`, + }, + { + name: "Set Empty JSON Array", + setCmd: `JSON.SET emptyArray $ []`, + getCmd: `JSON.GET emptyArray`, + expected: `[]`, + }, + { + name: "Set JSON with Unicode", + setCmd: `JSON.SET unicode $ ` + unicodeJSON, + getCmd: `JSON.GET unicode`, + expected: unicodeJSON, + }, + { + name: "Set JSON with Escaped Characters", + setCmd: `JSON.SET escaped $ ` + escapedCharsJSON, + getCmd: `JSON.GET escaped`, + expected: escapedCharsJSON, + }, + { + name: "Set and Get Complex JSON", + setCmd: `JSON.SET inventory $ ` + complexJSON, + getCmd: `JSON.GET inventory`, + expected: complexJSON, + }, + { + name: "Get Nested Array", + setCmd: `JSON.SET inventory $ ` + complexJSON, + getCmd: `JSON.GET inventory $.inventory.mountain_bikes[*].model`, + expected: `["Phoebe","Quaoar","Weywot"]`, + }, + { + name: "Get Nested Object", + setCmd: `JSON.SET inventory $ ` + complexJSON, + getCmd: `JSON.GET inventory $.inventory.mountain_bikes[0].specs`, + expected: `{"material":"carbon","weight":13.1}`, + }, + { + name: "Set Nested Value", + setCmd: `JSON.SET inventory $.inventory.mountain_bikes[0].price 2000`, + getCmd: `JSON.GET inventory $.inventory.mountain_bikes[0].price`, + expected: `2000`, + }, + { + name: "Get JSON with non-existent path", + setCmd: `JSON.SET user $ ` + simpleJSON, + getCmd: `JSON.GET user $.nonExistent`, + expected: `(nil)`, + }, + } + + // Multiple test cases will address JSON operations where the order of elements can vary, but all orders are "valid" and to be accepted + // The variation in order is due to the inherent nature of JSON objects. + // When dealing with a resultant array that contains elements from multiple objects, the order of these elements can have several valid permutations. + // Which then means, the overall order of elements in the resultant array is not fixed, although each sub-array within it is guaranteed to be ordered. + // What goes here: + // - Cases where the possible order of the resultant array is multiple permutations + // ref: https://github.com/DiceDB/dice/pull/365 + multipleOrderedTestCases := []struct { + name string + setCmd string + getCmd string + expected []string + }{ + { + name: "Get All Prices", + setCmd: `JSON.SET inventory $ ` + complexJSON, + getCmd: `JSON.GET inventory $..price`, + expected: []string{`[1475,3941,1920,2072,3264]`, `[1920,2072,3264,1475,3941]`}, // Ordering agnostic + }, + { + name: "Set Multiple Nested Values", + setCmd: `JSON.SET inventory $.inventory.*[?(@.price<2000)].price 1500`, + getCmd: `JSON.GET inventory $..price`, + expected: []string{`[1500,3941,1500,2072,3264]`, `[1500,2072,3264,1500,3941]`}, // Ordering agnostic + }, + } + + t.Run("Single Ordered Test Cases", func(t *testing.T) { + for _, tc := range singleOrderedTestCases { + t.Run(tc.name, func(t *testing.T) { + if tc.setCmd != "" { + result := FireCommand(conn, tc.setCmd) + assert.Equal(t, "OK", result) + } + + if tc.getCmd != "" { + result := FireCommand(conn, tc.getCmd) + if testutils.IsJSONResponse(result.(string)) { + assert.JSONEq(t, tc.expected, result.(string)) + } else { + assert.Equal(t, tc.expected, result) + } + } + }) + } + }) + + t.Run("Multiple Ordered Test Cases", func(t *testing.T) { + for _, tc := range multipleOrderedTestCases { + t.Run(tc.name, func(t *testing.T) { + if tc.setCmd != "" { + result := FireCommand(conn, tc.setCmd) + assert.Equal(t, "OK", result) + } + + if tc.getCmd != "" { + result := FireCommand(conn, tc.getCmd) + testutils.AssertJSONEqualList(t, tc.expected, result.(string)) + } + }) + } + }) +} + +func TestJSONSetWithInvalidJSON(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + + testCases := []struct { + name string + command string + expected string + }{ + { + name: "Set Invalid JSON", + command: `JSON.SET invalid $ {invalid:json}`, + expected: "ERR invalid JSON", + }, + { + name: "Set JSON with Wrong Number of Arguments", + command: `JSON.SET`, + expected: "ERR wrong number of arguments for 'json.set' command", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := FireCommand(conn, tc.command) + assert.True(t, strings.HasPrefix(result.(string), tc.expected), fmt.Sprintf("Expected: %s, Got: %s", tc.expected, result)) + }) + } +} + +func TestUnsupportedJSONPathPatterns(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + complexJSON := `{"inventory":{"mountain_bikes":[{"id":"bike:1","model":"Phoebe","price":1920,"specs":{"material":"carbon","weight":13.1},"colors":["black","silver"]},{"id":"bike:2","model":"Quaoar","price":2072,"specs":{"material":"aluminium","weight":7.9},"colors":["black","white"]},{"id":"bike:3","model":"Weywot","price":3264,"specs":{"material":"alloy","weight":13.8}}],"commuter_bikes":[{"id":"bike:4","model":"Salacia","price":1475,"specs":{"material":"aluminium","weight":16.6},"colors":["black","silver"]},{"id":"bike:5","model":"Mimas","price":3941,"specs":{"material":"alloy","weight":11.6}}]}}` + + setupCmd := `JSON.SET bikes:inventory $ ` + complexJSON + result := FireCommand(conn, setupCmd) + assert.Equal(t, "OK", result) + + testCases := []struct { + name string + command string + expected string + }{ + { + name: "Regex in JSONPath", + command: `JSON.GET bikes:inventory '$..[?(@.specs.material =~ "(?i)al")].model'`, + expected: "ERR invalid JSONPath", + }, + { + name: "Using @ for referencing other fields", + command: `JSON.GET bikes:inventory '$.inventory.mountain_bikes[?(@.specs.material =~ @.regex_pat)].model'`, + expected: "ERR invalid JSONPath", + }, + { + name: "Complex condition with multiple comparisons", + command: `JSON.GET bikes:inventory '$..mountain_bikes[?(@.price < 3000 && @.specs.weight < 10)]'`, + expected: "ERR invalid JSONPath", + }, + { + name: "Get all colors", + command: `JSON.GET bikes:inventory '$..[*].colors'`, + expected: "ERR invalid JSONPath", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := FireCommand(conn, tc.command) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestJSONSetWithNXAndXX(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + + FireCommand(conn, "DEL user") + + user1 := `{"name":"John","age":30}` + user2 := `{"name":"Rahul","age":28}` + + testCases := []struct { + name string + commands []string + expected []interface{} + }{ + { + name: "Set with XX on non-existent key", + commands: []string{"JSON.SET user $ " + user1 + " XX", "JSON.SET user $ " + user1 + " NX", "JSON.GET user"}, + expected: []interface{}{"(nil)", "OK", user1}, + }, + { + name: "Set with NX on existing key", + commands: []string{"DEL user", "JSON.SET user $ " + user1, "JSON.SET user $ " + user1 + " NX"}, + expected: []interface{}{int64(1), "OK", "(nil)"}, + }, + { + name: "Set with XX on existing key", + commands: []string{"JSON.SET user $ " + user2 + " XX", "JSON.GET user", "DEL user"}, + expected: []interface{}{"OK", user2, int64(1)}, + }, + { + name: "Set with NX on non-existent key", + commands: []string{"JSON.SET user $ " + user2 + " NX", "JSON.SET user $ " + user1 + " NX", "JSON.GET user"}, + expected: []interface{}{"OK", "(nil)", user2}, + }, + { + name: "Invalid combinations of NX and XX", + commands: []string{"JSON.SET user $ " + user2 + " NX NX", "JSON.SET user $ " + user2 + " NX XX", "JSON.SET user $ " + user2 + " NX Abcd", "JSON.SET user $ " + user2 + " NX "}, + expected: []interface{}{"ERR syntax error", "ERR syntax error", "ERR syntax error", "(nil)"}, + }, + { + name: "Invalid combinations of XX", + commands: []string{"JSON.SET user $ " + user2 + " XX XX", "JSON.SET user $ " + user2 + " XX NX", "JSON.SET user $ " + user2 + " XX Abcd", "JSON.SET user $ " + user2 + " XX "}, + expected: []interface{}{"ERR syntax error", "ERR syntax error", "ERR syntax error", "OK"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.commands { + result := FireCommand(conn, cmd) + jsonResult, isString := result.(string) + if isString && testutils.IsJSONResponse(jsonResult) { + assert.JSONEq(t, tc.expected[i].(string), jsonResult) + } else { + assert.Equal(t, tc.expected[i], result) + } + } + }) + } +} + func TestJSONDel(t *testing.T) { conn := getLocalConnection() defer conn.Close() - preTestChecksCommand := "DEL user" - postTestChecksCommand := "DEL user" + FireCommand(conn, "DEL user") + + // preTestChecksCommand := "DEL user" + // postTestChecksCommand := "DEL user" testCases := []IntegrationTestCase{ { @@ -167,7 +508,111 @@ func TestJSONDel(t *testing.T) { }, } - runIntegrationTests(t, conn, testCases, preTestChecksCommand, postTestChecksCommand) + runIntegrationTests(t, conn, testCases, "", "") +} + +func TestJSONMGET(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + defer FireCommand(conn, "DEL xx yy zz p q r t u v doc1 doc2") + + setupData := map[string]string{ + "xx": `["hehhhe","hello"]`, + "yy": `{"name":"jerry","partner":{"name":"jerry","language":["rust"]},"partner2":{"language":["rust"]}}`, + "zz": `{"name":"tom","partner":{"name":"tom","language":["rust"]},"partner2":{"age":12,"language":["rust"]}}`, + "doc1": `{"a":1,"b":2,"nested":{"a":3},"c":null}`, + "doc2": `{"a":4,"b":5,"nested":{"a":6},"c":null}`, + } + + for key, value := range setupData { + resp := FireCommand(conn, fmt.Sprintf("JSON.SET %s $ %s", key, value)) + assert.Equal(t, "OK", resp) + } + + testCases := []struct { + name string + command string + expected []interface{} + }{ + { + name: "MGET with root path", + command: "JSON.MGET xx yy zz $", + expected: []interface{}{setupData["xx"], setupData["yy"], setupData["zz"]}, + }, + { + name: "MGET with specific path", + command: "JSON.MGET xx yy zz $.name", + expected: []interface{}{"(nil)", `"jerry"`, `"tom"`}, + }, + { + name: "MGET with nested path", + command: "JSON.MGET xx yy zz $.partner2.age", + expected: []interface{}{"(nil)", "(nil)", "12"}, + }, + { + name: "MGET error", + command: "JSON.MGET t", + expected: []interface{}{"ERR wrong number of arguments for 'json.mget' command"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := FireCommand(conn, tc.command) + results, ok := result.([]interface{}) + if ok { + assert.Equal(t, len(tc.expected), len(results)) + for i := range results { + if testutils.IsJSONResponse(tc.expected[i].(string)) { + assert.JSONEq(t, tc.expected[i].(string), results[i].(string)) + } else { + assert.Equal(t, tc.expected[i], results[i]) + } + } + } else { + assert.Equal(t, tc.expected[0], result) + } + }) + } + + t.Run("MGET with recursive path", testJSONMGETRecursive(conn)) +} + +func sliceContainsItem(slice []int, item int) bool { + for _, v := range slice { + if v == item { + return true + } + } + return false +} + +func testJSONMGETRecursive(conn net.Conn) func(*testing.T) { + return func(t *testing.T) { + result := FireCommand(conn, "JSON.MGET doc1 doc2 $..a") + results, ok := result.([]interface{}) + assert.True(t, ok, "Expected result to be a slice of interface{}") + assert.Equal(t, 2, len(results), "Expected 2 results") + + expectedSets := [][]int{ + {1, 3}, + {4, 6}, + } + + for i, res := range results { + var actualSet []int + err := sonic.UnmarshalString(res.(string), &actualSet) + assert.Nil(t, err, "Failed to unmarshal JSON") + + assert.True(t, len(actualSet) == len(expectedSets[i]), + "Mismatch in number of elements for set %d", i) + + for _, expected := range expectedSets[i] { + assert.True(t, sliceContainsItem(actualSet, expected), + "Set %d does not contain expected value %d", i, expected) + } + } + } } func TestJSONForget(t *testing.T) { @@ -545,6 +990,66 @@ func TestJsonStrlen(t *testing.T) { } } +func TestJsonSTRAPPEND(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + + simpleJSON := `{"name":"John","age":30}` + + testCases := []struct { + name string + setCmd string + getCmd string + expected interface{} + }{ + { + name: "STRAPPEND to nested string", + setCmd: `JSON.SET doc $ ` + simpleJSON, + getCmd: `JSON.STRAPPEND doc $.name " Doe"`, + expected: int64(8), + }, + { + name: "STRAPPEND to multiple paths", + setCmd: `JSON.SET doc $ ` + simpleJSON, + getCmd: `JSON.STRAPPEND doc $..name "baz"`, + expected: int64(7), + }, + { + name: "STRAPPEND to non-string", + setCmd: `JSON.SET doc $ ` + simpleJSON, + getCmd: `JSON.STRAPPEND doc $.age " years"`, + expected: "(nil)", + }, + { + name: "STRAPPEND with empty string", + setCmd: `JSON.SET doc $ ` + simpleJSON, + getCmd: `JSON.STRAPPEND doc $.name ""`, + expected: int64(4), + }, + { + name: "STRAPPEND to non-existent path", + setCmd: `JSON.SET doc $ ` + simpleJSON, + getCmd: `JSON.STRAPPEND doc $.nonexistent " test"`, + expected: []interface{}{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Use FLUSHDB to clear all keys before each test + result := FireCommand(conn, "FLUSHDB") + assert.Equal(t, "OK", result) + + result = FireCommand(conn, tc.setCmd) + assert.Equal(t, "OK", result) + + result = FireCommand(conn, tc.getCmd) + assert.Equal(t, tc.expected, result) + + }) + } +} + func TestJSONClearOperations(t *testing.T) { conn := getLocalConnection() defer conn.Close() diff --git a/integration_tests/commands/async/mget_test.go b/integration_tests/commands/resp/mget_test.go similarity index 99% rename from integration_tests/commands/async/mget_test.go rename to integration_tests/commands/resp/mget_test.go index 715c372ee..2df676e08 100644 --- a/integration_tests/commands/async/mget_test.go +++ b/integration_tests/commands/resp/mget_test.go @@ -1,4 +1,4 @@ -package async +package resp import ( "testing" diff --git a/integration_tests/commands/async/mset_test.go b/integration_tests/commands/resp/mset_test.go similarity index 99% rename from integration_tests/commands/async/mset_test.go rename to integration_tests/commands/resp/mset_test.go index d0d8ba647..0e82a179a 100644 --- a/integration_tests/commands/async/mset_test.go +++ b/integration_tests/commands/resp/mset_test.go @@ -1,4 +1,4 @@ -package async +package resp import ( "testing" diff --git a/integration_tests/commands/websocket/deque_test.go b/integration_tests/commands/websocket/deque_test.go index 1aa8e84e6..f3916c9d3 100644 --- a/integration_tests/commands/websocket/deque_test.go +++ b/integration_tests/commands/websocket/deque_test.go @@ -26,7 +26,7 @@ func deqRandStr(n int) string { return string(b) } -func deqTestInit() { +func deqTestInit() (deqNormalValues, deqEdgeValues []string) { randSeed := time.Now().UnixNano() deqRandGenerator = rand.New(rand.NewSource(randSeed)) fmt.Printf("rand seed: %v", randSeed) @@ -67,10 +67,11 @@ func deqTestInit() { "-9223372036854775808", // min 64 bit int "9223372036854775807", // max 64 bit int } + return deqNormalValues, deqEdgeValues } func TestLPush(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewWebsocketCommandExecutor() testCases := []struct { @@ -111,7 +112,7 @@ func TestLPush(t *testing.T) { } func TestRPush(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewWebsocketCommandExecutor() testCases := []struct { @@ -152,7 +153,7 @@ func TestRPush(t *testing.T) { } func TestLPushLPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewWebsocketCommandExecutor() getPops := func(values []string) []string { @@ -208,7 +209,7 @@ func TestLPushLPop(t *testing.T) { } func TestLPushRPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewWebsocketCommandExecutor() getPops := func(values []string) []string { @@ -264,7 +265,7 @@ func TestLPushRPop(t *testing.T) { } func TestRPushLPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewWebsocketCommandExecutor() getPops := func(values []string) []string { @@ -320,7 +321,7 @@ func TestRPushLPop(t *testing.T) { } func TestRPushRPop(t *testing.T) { - deqTestInit() + deqNormalValues, deqEdgeValues := deqTestInit() exec := NewWebsocketCommandExecutor() getPops := func(values []string) []string { @@ -376,7 +377,6 @@ func TestRPushRPop(t *testing.T) { } func TestLRPushLRPop(t *testing.T) { - deqTestInit() exec := NewWebsocketCommandExecutor() testCases := []struct { @@ -417,7 +417,6 @@ func TestLRPushLRPop(t *testing.T) { } func TestLLEN(t *testing.T) { - deqTestInit() exec := NewWebsocketCommandExecutor() testCases := []struct { diff --git a/internal/cmd/cmds.go b/internal/cmd/cmds.go index a561764be..d2a955c67 100644 --- a/internal/cmd/cmds.go +++ b/internal/cmd/cmds.go @@ -5,11 +5,36 @@ import ( "strings" "github.com/dgryski/go-farm" + "github.com/dicedb/dice/internal/object" ) +// DiceDBCmd represents a command structure to be executed +// within a DiceDB system. This struct emulates the way DiceDB commands +// are structured, including the command itself, additional arguments, +// and an optional object to store or manipulate. type DiceDBCmd struct { - Cmd string + // Cmd represents the command to execute (e.g., "SET", "GET", "DEL"). + // This is the main command keyword that specifies the action to perform + // in DiceDB. For example: + // - "SET": To store a value. + // - "GET": To retrieve a value. + // - "DEL": To delete a value. + // - "EXPIRE": To set a time-to-live for a key. + Cmd string + + // Args holds any additional parameters required by the command. + // For example: + // - If Cmd is "SET", Args might contain ["key", "value"]. + // - If Cmd is "EXPIRE", Args might contain ["key", "seconds"]. + // This slice allows flexible support for commands with variable arguments. Args []string + + // InternalObj is a pointer to an InternalObj, representing an optional data structure + // associated with the command. This contains pointer to the underlying simple + // types such as int, string or even complex types + // like hashes, sets, or sorted sets, which are stored and manipulated as objects. + // WARN: This parameter should be used with caution + InternalObj *object.InternalObj } type RedisCmds struct { diff --git a/internal/errors/migrated_errors.go b/internal/errors/migrated_errors.go index ac0176346..fd1b000bd 100644 --- a/internal/errors/migrated_errors.go +++ b/internal/errors/migrated_errors.go @@ -73,3 +73,11 @@ var ( return fmt.Errorf("ERR wrong type of path value - expected %s but found %s", expectedType, actualType) // Signals an unexpected type received when an integer was expected. } ) + +type PreProcessError struct { + Result interface{} +} + +func (e *PreProcessError) Error() string { + return fmt.Sprintf("%v", e.Result) +} diff --git a/internal/eval/commands.go b/internal/eval/commands.go index 3be4e8fd0..3ff1bb931 100644 --- a/internal/eval/commands.go +++ b/internal/eval/commands.go @@ -3,6 +3,7 @@ package eval import ( "strings" + "github.com/dicedb/dice/internal/cmd" dstore "github.com/dicedb/dice/internal/store" ) @@ -29,6 +30,15 @@ type DiceCmdMeta struct { // will utilize this function for evaluation, allowing for better handling of // complex command execution scenarios and improved response consistency. NewEval func([]string, *dstore.Store) *EvalResponse + + // StoreObjectEval is a specialized evaluation function for commands that operate on an object. + // It is designed for scenarios where the command and subsequent dependent command requires + // an object as part of its execution. This function processes the command, + // evaluates it based on the provided object, and returns an EvalResponse struct + // Commands that involve object manipulation, is not recommended for general use. + // Only commands that really requires full object definition to pass across multiple shards + // should implement this function. e.g. COPY, RENAME etc + StoreObjectEval func(*cmd.DiceDBCmd, *dstore.Store) *EvalResponse } type KeySpecs struct { @@ -38,8 +48,11 @@ type KeySpecs struct { } var ( - DiceCmds = map[string]DiceCmdMeta{} + PreProcessing = map[string]func([]string, *dstore.Store) *EvalResponse{} + DiceCmds = map[string]DiceCmdMeta{} +) +var ( echoCmdMeta = DiceCmdMeta{ Name: "ECHO", Info: `ECHO returns the string given as argument.`, @@ -125,9 +138,10 @@ var ( Sets a JSON value at the specified key. Returns OK if successful. Returns encoded error message if the number of arguments is incorrect or the JSON string is invalid.`, - Eval: evalJSONSET, - Arity: -3, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalJSONSET, + Arity: -3, + KeySpecs: KeySpecs{BeginIndex: 1}, + IsMigrated: true, } jsongetCmdMeta = DiceCmdMeta{ Name: "JSON.GET", @@ -135,9 +149,10 @@ var ( Returns the encoded RESP value of the key, if present Null reply: If the key doesn't exist or has expired. Error reply: If the number of arguments is incorrect or the stored value is not a JSON type.`, - Eval: evalJSONGET, - Arity: 2, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalJSONGET, + Arity: 2, + KeySpecs: KeySpecs{BeginIndex: 1}, + IsMigrated: true, } jsonMGetCmdMeta = DiceCmdMeta{ Name: "JSON.MGET", @@ -161,10 +176,10 @@ var ( 1.String ("true"/"false") that represents the resulting Boolean value. 2.NONEXISTENT if the document key does not exist. 3.WRONGTYPE error if the value at the path is not a Boolean value.`, + NewEval: evalJSONTOGGLE, Arity: 2, KeySpecs: KeySpecs{BeginIndex: 1}, IsMigrated: true, - NewEval: evalJSONTOGGLE, } jsontypeCmdMeta = DiceCmdMeta{ Name: "JSON.TYPE", @@ -172,9 +187,10 @@ var ( Returns string reply for each path, specified as the value's type. Returns RespNIL If the key doesn't exist. Error reply: If the number of arguments is incorrect.`, - Eval: evalJSONTYPE, - Arity: -2, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalJSONTYPE, + Arity: -2, + KeySpecs: KeySpecs{BeginIndex: 1}, + IsMigrated: true, } jsonclearCmdMeta = DiceCmdMeta{ Name: "JSON.CLEAR", @@ -193,10 +209,10 @@ var ( Returns an integer reply specified as the number of paths deleted (0 or more). Returns RespZero if the key doesn't exist or key is expired. Error reply: If the number of arguments is incorrect.`, + NewEval: evalJSONDEL, Arity: -2, KeySpecs: KeySpecs{BeginIndex: 1}, IsMigrated: true, - NewEval: evalJSONDEL, } jsonarrappendCmdMeta = DiceCmdMeta{ Name: "JSON.ARRAPPEND", @@ -213,10 +229,10 @@ var ( Returns an integer reply specified as the number of paths deleted (0 or more). Returns RespZero if the key doesn't exist or key is expired. Error reply: If the number of arguments is incorrect.`, + NewEval: evalJSONFORGET, Arity: -2, KeySpecs: KeySpecs{BeginIndex: 1}, IsMigrated: true, - NewEval: evalJSONFORGET, } jsonarrlenCmdMeta = DiceCmdMeta{ Name: "JSON.ARRLEN", @@ -233,10 +249,10 @@ var ( Name: "JSON.NUMMULTBY", Info: `JSON.NUMMULTBY key path value Multiply the number value stored at the specified path by a value.`, + NewEval: evalJSONNUMMULTBY, Arity: 3, KeySpecs: KeySpecs{BeginIndex: 1}, IsMigrated: true, - NewEval: evalJSONNUMMULTBY, } jsonobjlenCmdMeta = DiceCmdMeta{ Name: "JSON.OBJLEN", @@ -289,9 +305,10 @@ var ( the generated key is then used to store the provided JSON value at specified path. Returns unique identifier if successful. Returns encoded error message if the number of arguments is incorrect or the JSON string is invalid.`, - Eval: evalJSONINGEST, - Arity: -3, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalJSONINGEST, + Arity: -3, + KeySpecs: KeySpecs{BeginIndex: 1}, + IsMigrated: true, } jsonarrinsertCmdMeta = DiceCmdMeta{ Name: "JSON.ARRINSERT", @@ -514,16 +531,6 @@ var ( Eval: nil, Arity: 1, } - MultiCmdMeta = DiceCmdMeta{ - Name: "MULTI", - Info: `MULTI marks the start of the transaction for the client. - All subsequent commands fired will be queued for atomic execution. - The commands will not be executed until EXEC is triggered. - Once EXEC is triggered it executes all the commands in queue, - and closes the MULTI transaction.`, - Eval: evalMULTI, - Arity: 1, - } ExecCmdMeta = DiceCmdMeta{ Name: "EXEC", Info: `EXEC executes commands in a transaction, which is initiated by MULTI`, @@ -617,10 +624,12 @@ var ( } keysCmdMeta = DiceCmdMeta{ - Name: "KEYS", - Info: "KEYS command is used to get all the keys in the database. Complexity is O(n) where n is the number of keys in the database.", - Eval: evalKeys, + Name: "KEYS", + Info: "KEYS command is used to get all the keys in the database. Complexity is O(n) where n is the number of keys in the database.", + Eval: evalKeys, + Arity: 1, } + MGetCmdMeta = DiceCmdMeta{ Name: "MGET", Info: `The MGET command returns an array of RESP values corresponding to the provided keys. @@ -636,12 +645,24 @@ var ( Info: "PERSIST removes the expiration from a key", Eval: evalPersist, } + + //TODO: supports only http protocol, needs to be removed once http is migrated to multishard copyCmdMeta = DiceCmdMeta{ Name: "COPY", Info: `COPY command copies the value stored at the source key to the destination key.`, Eval: evalCOPY, Arity: -2, } + + //TODO: supports only http protocol, needs to be removed once http is migrated to multishard + objectCopyCmdMeta = DiceCmdMeta{ + Name: "OBJECTCOPY", + Info: `COPY command copies the value stored at the source key to the destination key.`, + StoreObjectEval: evalCOPYObject, + IsMigrated: true, + Arity: -2, + } + decrCmdMeta = DiceCmdMeta{ Name: "DECR", Info: `DECR decrements the value of the specified key in args by 1, @@ -770,9 +791,10 @@ var ( Name: "HGETALL", Info: `Returns all fields and values of the hash stored at key. In the returned value, every field name is followed by its value, so the length of the reply is twice the size of the hash.`, - Eval: evalHGETALL, - Arity: -2, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalHGETALL, + Arity: -2, + KeySpecs: KeySpecs{BeginIndex: 1}, + IsMigrated: true, } hValsCmdMeta = DiceCmdMeta{ Name: "HVALS", @@ -1043,26 +1065,28 @@ var ( jsonnumincrbyCmdMeta = DiceCmdMeta{ Name: "JSON.NUMINCRBY", Info: `Increment the number value stored at path by number.`, + NewEval: evalJSONNUMINCRBY, Arity: 3, KeySpecs: KeySpecs{BeginIndex: 1}, IsMigrated: true, - NewEval: evalJSONNUMINCRBY, } dumpkeyCMmdMeta = DiceCmdMeta{ Name: "DUMP", Info: `Serialize the value stored at key in a Redis-specific format and return it to the user. The returned value can be synthesized back into a Redis key using the RESTORE command.`, - Eval: evalDUMP, - Arity: 1, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalDUMP, + IsMigrated: true, + Arity: 1, + KeySpecs: KeySpecs{BeginIndex: 1}, } restorekeyCmdMeta = DiceCmdMeta{ Name: "RESTORE", Info: `Serialize the value stored at key in a Redis-specific format and return it to the user. The returned value can be synthesized back into a Redis key using the RESTORE command.`, - Eval: evalRestore, - Arity: 2, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalRestore, + IsMigrated: true, + Arity: 2, + KeySpecs: KeySpecs{BeginIndex: 1}, } typeCmdMeta = DiceCmdMeta{ Name: "TYPE", @@ -1277,9 +1301,10 @@ var ( Append the JSON string values to the string at path Returns an array of integer replies for each path, the string's new length, or nil, if the matching JSON value is not a string. Error reply: If the value at path is not a string or if the key doesn't exist.`, - Eval: evalJSONSTRAPPEND, - Arity: 3, - KeySpecs: KeySpecs{BeginIndex: 1}, + NewEval: evalJSONSTRAPPEND, + IsMigrated: true, + Arity: 3, + KeySpecs: KeySpecs{BeginIndex: 1}, } cmsInitByDimCmdMeta = DiceCmdMeta{ Name: "CMS.INITBYDIM", @@ -1375,6 +1400,9 @@ var ( ) func init() { + PreProcessing["COPY"] = evalGetObject + PreProcessing["RENAME"] = evalGET + DiceCmds["ABORT"] = abortCmdMeta DiceCmds["APPEND"] = appendCmdMeta DiceCmds["AUTH"] = authCmdMeta @@ -1398,6 +1426,7 @@ func init() { DiceCmds["COMMAND|DOCS"] = commandDocsCmdMeta DiceCmds["COMMAND|GETKEYSANDFLAGS"] = commandGetKeysAndFlagsCmdMeta DiceCmds["COPY"] = copyCmdMeta + DiceCmds["OBJECTCOPY"] = objectCopyCmdMeta DiceCmds["DBSIZE"] = dbSizeCmdMeta DiceCmds["DECR"] = decrCmdMeta DiceCmds["DECRBY"] = decrByCmdMeta @@ -1469,7 +1498,6 @@ func init() { DiceCmds["LRU"] = lruCmdMeta DiceCmds["MGET"] = MGetCmdMeta DiceCmds["MSET"] = msetCmdMeta - DiceCmds["MULTI"] = MultiCmdMeta DiceCmds["OBJECT"] = objectCmdMeta DiceCmds["PERSIST"] = persistCmdMeta DiceCmds["PFADD"] = pfAddCmdMeta diff --git a/internal/eval/dump_restore.go b/internal/eval/dump_restore.go index 6b09ae329..0c1428127 100644 --- a/internal/eval/dump_restore.go +++ b/internal/eval/dump_restore.go @@ -2,68 +2,13 @@ package eval import ( "bytes" - "encoding/base64" "encoding/binary" "errors" "hash/crc64" - "strconv" - "github.com/dicedb/dice/internal/clientio" - diceerrors "github.com/dicedb/dice/internal/errors" "github.com/dicedb/dice/internal/object" - dstore "github.com/dicedb/dice/internal/store" ) -func evalDUMP(args []string, store *dstore.Store) []byte { - if len(args) < 1 { - return diceerrors.NewErrArity("DUMP") - } - key := args[0] - obj := store.Get(key) - if obj == nil { - return diceerrors.NewErrWithFormattedMessage("nil") - } - - serializedValue, err := rdbSerialize(obj) - if err != nil { - return diceerrors.NewErrWithMessage("serialization failed") - } - encodedResult := base64.StdEncoding.EncodeToString(serializedValue) - return clientio.Encode(encodedResult, false) -} - -func evalRestore(args []string, store *dstore.Store) []byte { - if len(args) < 3 { - return diceerrors.NewErrArity("RESTORE") - } - - key := args[0] - ttlStr := args[1] - ttl, _ := strconv.ParseInt(ttlStr, 10, 64) - - encodedValue := args[2] - serializedData, err := base64.StdEncoding.DecodeString(encodedValue) - if err != nil { - return diceerrors.NewErrWithMessage("failed to decode base64 value") - } - - obj, err := rdbDeserialize(serializedData) - if err != nil { - return diceerrors.NewErrWithMessage("deserialization failed: " + err.Error()) - } - - newobj := store.NewObj(obj.Value, ttl, obj.TypeEncoding, obj.TypeEncoding) - var keepttl = true - - if ttl > 0 { - store.Put(key, newobj, dstore.WithKeepTTL(keepttl)) - } else { - store.Put(key, obj) - } - - return clientio.RespOK -} - func rdbDeserialize(data []byte) (*object.Obj, error) { if len(data) < 3 { return nil, errors.New("insufficient data for deserialization") diff --git a/internal/eval/eval.go b/internal/eval/eval.go index c80b8de61..37460356e 100644 --- a/internal/eval/eval.go +++ b/internal/eval/eval.go @@ -14,7 +14,6 @@ import ( "github.com/dicedb/dice/internal/eval/geo" "github.com/dicedb/dice/internal/eval/sortedset" "github.com/dicedb/dice/internal/object" - "github.com/rs/xid" "github.com/dicedb/dice/internal/sql" @@ -264,90 +263,33 @@ func adjustIndex(idx int, arr []any) int { return idx } -// evalJSONTYPE retrieves a JSON value type stored at the specified key -// args must contain at least the key; (path unused in this implementation) -// Returns response.RespNIL if key is expired, or it does not exist +// evalJSONMGET retrieves a JSON value stored for the multiple key +// args must contain at least the key and a path; // Returns encoded error response if incorrect number of arguments -// The RESP value of the key's value type is encoded and then returned -func evalJSONTYPE(args []string, store *dstore.Store) []byte { - if len(args) < 1 { - return diceerrors.NewErrArity("JSON.TYPE") - } - key := args[0] - - // Default path is root if not specified - path := defaultRootPath - if len(args) > 1 { - path = args[1] - } - // Retrieve the object from the database - obj := store.Get(key) - if obj == nil { - return clientio.RespNIL - } - - errWithMessage := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeJSON, object.ObjEncodingJSON) - if errWithMessage != nil { - return errWithMessage - } - - jsonData := obj.Value - - if path == defaultRootPath { - _, err := sonic.Marshal(jsonData) - if err != nil { - return diceerrors.NewErrWithMessage("could not serialize result") - } - // If path is root and len(args) == 1, return "object" instantly - if len(args) == 1 { - return clientio.Encode(utils.ObjectType, false) - } - } - - // Parse the JSONPath expression - expr, err := jp.ParseString(path) - if err != nil { - return diceerrors.NewErrWithMessage("invalid JSONPath") +// The RESP value of the key is encoded and then returned +// TODO: Needs to be removed after http and websocket migrated to the multithreading +func evalJSONMGET(args []string, store *dstore.Store) []byte { + if len(args) < 2 { + return diceerrors.NewErrArity("JSON.MGET") } - results := expr.Get(jsonData) - if len(results) == 0 { - return clientio.RespEmptyArray - } + var results []interface{} - typeList := make([]string, 0, len(results)) - for _, result := range results { - jsonType := utils.GetJSONFieldType(result) - typeList = append(typeList, jsonType) - } - return clientio.Encode(typeList, false) -} + // Default path is root if not specified + argsLen := len(args) + path := args[argsLen-1] -// evalJSONGET retrieves a JSON value stored at the specified key -// args must contain at least the key; (path unused in this implementation) -// Returns response.RespNIL if key is expired, or it does not exist -// Returns encoded error response if incorrect number of arguments -// The RESP value of the key is encoded and then returned -func evalJSONGET(args []string, store *dstore.Store) []byte { - if len(args) < 1 { - return diceerrors.NewErrArity("JSON.GET") + for i := 0; i < (argsLen - 1); i++ { + key := args[i] + result, _ := jsonMGETHelper(store, path, key) + results = append(results, result) } - key := args[0] - // Default path is root if not specified - path := defaultRootPath - if len(args) > 1 { - path = args[1] - } - result, err := jsonGETHelper(store, path, key) - if err != nil { - return err - } - return clientio.Encode(result, false) + var interfaceObj interface{} = results + return clientio.Encode(interfaceObj, false) } -// helper function used by evalJSONGET and evalJSONMGET to prepare the results -func jsonGETHelper(store *dstore.Store, path, key string) (result interface{}, err2 []byte) { +func jsonMGETHelper(store *dstore.Store, path, key string) (result interface{}, err2 []byte) { // Retrieve the object from the database obj := store.Get(key) if obj == nil { @@ -396,31 +338,6 @@ func jsonGETHelper(store *dstore.Store, path, key string) (result interface{}, e return string(resultBytes), nil } -// evalJSONMGET retrieves a JSON value stored for the multiple key -// args must contain at least the key and a path; -// Returns encoded error response if incorrect number of arguments -// The RESP value of the key is encoded and then returned -func evalJSONMGET(args []string, store *dstore.Store) []byte { - if len(args) < 2 { - return diceerrors.NewErrArity("JSON.MGET") - } - - var results []interface{} - - // Default path is root if not specified - argsLen := len(args) - path := args[argsLen-1] - - for i := 0; i < (argsLen - 1); i++ { - key := args[i] - result, _ := jsonGETHelper(store, path, key) - results = append(results, result) - } - - var interfaceObj interface{} = results - return clientio.Encode(interfaceObj, false) -} - // ReverseSlice takes a slice of any type and returns a new slice with the elements reversed. func ReverseSlice[T any](slice []T) []T { reversed := make([]T, len(slice)) @@ -430,96 +347,6 @@ func ReverseSlice[T any](slice []T) []T { return reversed } -// evalJSONSET stores a JSON value at the specified key -// args must contain at least the key, path (unused in this implementation), and JSON string -// Returns encoded error response if incorrect number of arguments -// Returns encoded error if the JSON string is invalid -// Returns response.RespOK if the JSON value is successfully stored -func evalJSONSET(args []string, store *dstore.Store) []byte { - // Check if there are enough arguments - if len(args) < 3 { - return diceerrors.NewErrArity("JSON.SET") - } - - key := args[0] - path := args[1] - jsonStr := args[2] - for i := 3; i < len(args); i++ { - switch strings.ToUpper(args[i]) { - case NX: - if i != len(args)-1 { - return diceerrors.NewErrWithMessage(diceerrors.SyntaxErr) - } - obj := store.Get(key) - if obj != nil { - return clientio.RespNIL - } - case XX: - if i != len(args)-1 { - return diceerrors.NewErrWithMessage(diceerrors.SyntaxErr) - } - obj := store.Get(key) - if obj == nil { - return clientio.RespNIL - } - - default: - return diceerrors.NewErrWithMessage(diceerrors.SyntaxErr) - } - } - - // Parse the JSON string - var jsonValue interface{} - if err := sonic.UnmarshalString(jsonStr, &jsonValue); err != nil { - return diceerrors.NewErrWithFormattedMessage("invalid JSON: %v", err.Error()) - } - - // Retrieve existing object or create new one - obj := store.Get(key) - var rootData interface{} - - if obj == nil { - // If the key doesn't exist, create a new object - if path != defaultRootPath { - rootData = make(map[string]interface{}) - } else { - rootData = jsonValue - } - } else { - // If the key exists, check if it's a JSON object - err := object.AssertType(obj.TypeEncoding, object.ObjTypeJSON) - if err != nil { - return clientio.Encode(err, false) - } - err = object.AssertEncoding(obj.TypeEncoding, object.ObjEncodingJSON) - if err != nil { - return clientio.Encode(err, false) - } - rootData = obj.Value - } - - // If path is not root, use JSONPath to set the value - if path != defaultRootPath { - expr, err := jp.ParseString(path) - if err != nil { - return diceerrors.NewErrWithMessage("invalid JSONPath") - } - - err = expr.Set(rootData, jsonValue) - if err != nil { - return diceerrors.NewErrWithMessage("failed to set value") - } - } else { - // If path is root, replace the entire JSON - rootData = jsonValue - } - - // Create a new object with the updated JSON data - newObj := store.NewObj(rootData, -1, object.ObjTypeJSON, object.ObjEncodingJSON) - store.Put(key, newObj) - return clientio.RespOK -} - // Parses and returns the input string as an int64 or float64 func parseFloatInt(input string) (result interface{}, err error) { // Try to parse as an integer @@ -539,34 +366,6 @@ func parseFloatInt(input string) (result interface{}, err error) { return } -// evalJSONINGEST stores a value at a dynamically generated key -// The key is created using a provided key prefix combined with a unique identifier -// args must contains key_prefix and path and json value -// It will call to evalJSONSET internally. -// Returns encoded error response if incorrect number of arguments -// Returns encoded error if the JSON string is invalid -// Returns unique identifier if the JSON value is successfully stored -func evalJSONINGEST(args []string, store *dstore.Store) []byte { - if len(args) < 3 { - return diceerrors.NewErrArity("JSON.INGEST") - } - - keyPrefix := args[0] - - uniqueID := xid.New() - uniqueKey := keyPrefix + uniqueID.String() - - var setArgs []string - setArgs = append(setArgs, uniqueKey) - setArgs = append(setArgs, args[1:]...) - - result := evalJSONSET(setArgs, store) - if bytes.Equal(result, clientio.RespOK) { - return clientio.Encode(uniqueID.String(), true) - } - return result -} - // evalDEL deletes all the specified keys in args list // returns the count of total deleted keys after encoding func evalDEL(args []string, store *dstore.Store) []byte { @@ -645,15 +444,6 @@ func evalSLEEP(args []string, store *dstore.Store) []byte { return clientio.RespOK } -// evalMULTI marks the start of the transaction for the client. -// All subsequent commands fired will be queued for atomic execution. -// The commands will not be executed until EXEC is triggered. -// Once EXEC is triggered it executes all the commands in queue, -// and closes the MULTI transaction. -func evalMULTI(args []string, store *dstore.Store) []byte { - return clientio.RespOK -} - // EvalQWATCH adds the specified key to the watch list for the caller client. // Every time a key in the watch list is modified, the client will be sent a response // containing the new value of the key along with the operation that was performed on it. @@ -982,6 +772,7 @@ func evalCommandList(args []string) []byte { } // evalKeys returns the list of keys that match the pattern should be the only param in args +// TODO: Needs to be removed after http and websocket migrated to the multithreading func evalKeys(args []string, store *dstore.Store) []byte { if len(args) != 1 { return diceerrors.NewErrArity("KEYS") @@ -1088,6 +879,7 @@ func evalCommandDocs(args []string) []byte { return clientio.Encode(result, false) } +// TODO: Needs to be removed after http and websocket migrated to the multithreading func evalRename(args []string, store *dstore.Store) []byte { if len(args) != 2 { return diceerrors.NewErrArity("RENAME") @@ -1116,6 +908,7 @@ func evalRename(args []string, store *dstore.Store) []byte { // For each key, if the key is expired or does not exist, the response will be response.RespNIL; // otherwise, the response will be the RESP value of the key. // MGET is atomic, it retrieves all values at once +// TODO: Needs to be removed after http and websocket migrated to the multithreading func evalMGET(args []string, store *dstore.Store) []byte { if len(args) < 1 { return diceerrors.NewErrArity("MGET") @@ -1173,6 +966,7 @@ func evalPersist(args []string, store *dstore.Store) []byte { return clientio.RespOne } +// TODO: Needs to be removed after http and websocket migrated to the multithreading func evalCOPY(args []string, store *dstore.Store) []byte { if len(args) < 2 { return diceerrors.NewErrArity("COPY") @@ -1189,7 +983,7 @@ func evalCOPY(args []string, store *dstore.Store) []byte { for i := 2; i < len(args); i++ { arg := strings.ToUpper(args[i]) - if arg == "REPLACE" { + if arg == dstore.Replace { isReplace = true } } @@ -1222,32 +1016,6 @@ func evalCOPY(args []string, store *dstore.Store) []byte { return clientio.RespOne } -func evalHGETALL(args []string, store *dstore.Store) []byte { - if len(args) != 1 { - return diceerrors.NewErrArity("HGETALL") - } - - key := args[0] - - obj := store.Get(key) - - var hashMap HashMap - var results []string - - if obj != nil { - if err := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeHashMap, object.ObjEncodingHashMap); err != nil { - return diceerrors.NewErrWithMessage(diceerrors.WrongTypeErr) - } - hashMap = obj.Value.(HashMap) - } - - for hmKey, hmValue := range hashMap { - results = append(results, hmKey, hmValue) - } - - return clientio.Encode(results, false) -} - func evalObjectIdleTime(key string, store *dstore.Store) []byte { obj := store.GetNoTouch(key) if obj == nil { @@ -1449,6 +1217,7 @@ func evalSDIFF(args []string, store *dstore.Store) []byte { return clientio.Encode(members, false) } +// Migrated to the new eval, but kept for http and websocket func evalSINTER(args []string, store *dstore.Store) []byte { if len(args) < 1 { return diceerrors.NewErrArity("SINTER") @@ -1731,74 +1500,3 @@ func evalGEODIST(args []string, store *dstore.Store) []byte { return clientio.Encode(utils.RoundToDecimals(result, 4), false) } - -// evalJSONSTRAPPEND appends a string value to the JSON string value at the specified path -// in the JSON object saved at the key in arguments. -// Args must contain at least a key and the string value to append. -// If the key does not exist or is expired, it returns an error response. -// If the value at the specified path is not a string, it returns an error response. -// Returns the new length of the string at the specified path if successful. -func evalJSONSTRAPPEND(args []string, store *dstore.Store) []byte { - if len(args) != 3 { - return diceerrors.NewErrArity("JSON.STRAPPEND") - } - - key := args[0] - path := args[1] - value := args[2] - - obj := store.Get(key) - if obj == nil { - return diceerrors.NewErrWithMessage(diceerrors.NoKeyExistsErr) - } - - errWithMessage := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeJSON, object.ObjEncodingJSON) - if errWithMessage != nil { - return diceerrors.NewErrWithFormattedMessage(diceerrors.WrongKeyTypeErr) - } - - jsonData := obj.Value - - var resultsArray []interface{} - - if path == "$" { - // Handle root-level string - if str, ok := jsonData.(string); ok { - unquotedValue := strings.Trim(value, "\"") - newValue := str + unquotedValue - resultsArray = append(resultsArray, int64(len(newValue))) - jsonData = newValue - } else { - return clientio.RespEmptyArray - } - } else { - expr, err := jp.ParseString(path) - if err != nil { - return clientio.RespEmptyArray - } - - _, modifyErr := expr.Modify(jsonData, func(data any) (interface{}, bool) { - switch v := data.(type) { - case string: - unquotedValue := strings.Trim(value, "\"") - newValue := v + unquotedValue - resultsArray = append([]interface{}{int64(len(newValue))}, resultsArray...) - return newValue, true - default: - resultsArray = append([]interface{}{clientio.RespNIL}, resultsArray...) - return data, false - } - }) - - if modifyErr != nil { - return clientio.RespEmptyArray - } - } - - if len(resultsArray) == 0 { - return clientio.RespEmptyArray - } - - obj.Value = jsonData - return clientio.Encode(resultsArray, false) -} diff --git a/internal/eval/eval_test.go b/internal/eval/eval_test.go index 280abd02a..b7a92aba5 100644 --- a/internal/eval/eval_test.go +++ b/internal/eval/eval_test.go @@ -61,6 +61,7 @@ func TestEval(t *testing.T) { testEvalJSONTYPE(t, store) testEvalJSONGET(t, store) testEvalJSONSET(t, store) + testEvalJSONNUMINCRBY(t, store) testEvalJSONNUMMULTBY(t, store) testEvalJSONTOGGLE(t, store) testEvalJSONARRAPPEND(t, store) @@ -81,6 +82,7 @@ func TestEval(t *testing.T) { testEvalPFCOUNT(t, store) testEvalPFMERGE(t, store) testEvalHGET(t, store) + testEvalHGETALL(t, store) testEvalHMGET(t, store) testEvalHSTRLEN(t, store) testEvalHEXISTS(t, store) @@ -100,7 +102,6 @@ func TestEval(t *testing.T) { testEvalLRANGE(t, store) testEvalGETDEL(t, store) testEvalGETEX(t, store) - testEvalJSONNUMINCRBY(t, store) testEvalDUMP(t, store) testEvalTYPE(t, store) testEvalCOMMAND(t, store) @@ -1614,7 +1615,7 @@ func testEvalJSONDEL(t *testing.T, store *dstore.Store) { setup: func() {}, input: []string{"NONEXISTENT_KEY"}, migratedOutput: EvalResponse{ - Result: 0, + Result: clientio.IntegerZero, Error: nil, }, }, @@ -1631,7 +1632,7 @@ func testEvalJSONDEL(t *testing.T, store *dstore.Store) { }, input: []string{"EXISTING_KEY"}, migratedOutput: EvalResponse{ - Result: int64(1), + Result: 1, Error: nil, }, }, @@ -1649,7 +1650,7 @@ func testEvalJSONDEL(t *testing.T, store *dstore.Store) { input: []string{"EXISTING_KEY", "$..language"}, migratedOutput: EvalResponse{ - Result: int64(2), + Result: 2, Error: nil, }, }, @@ -1667,7 +1668,7 @@ func testEvalJSONDEL(t *testing.T, store *dstore.Store) { input: []string{"EXISTING_KEY", "$.*"}, migratedOutput: EvalResponse{ - Result: int64(6), + Result: 6, Error: nil, }, }, @@ -1691,7 +1692,7 @@ func testEvalJSONFORGET(t *testing.T, store *dstore.Store) { setup: func() {}, input: []string{"NONEXISTENT_KEY"}, migratedOutput: EvalResponse{ - Result: 0, + Result: clientio.IntegerZero, Error: nil, }, }, @@ -1708,7 +1709,7 @@ func testEvalJSONFORGET(t *testing.T, store *dstore.Store) { }, input: []string{"EXISTING_KEY"}, migratedOutput: EvalResponse{ - Result: int64(1), + Result: 1, Error: nil, }, }, @@ -1726,7 +1727,7 @@ func testEvalJSONFORGET(t *testing.T, store *dstore.Store) { input: []string{"EXISTING_KEY", "$..language"}, migratedOutput: EvalResponse{ - Result: int64(2), + Result: 2, Error: nil, }, }, @@ -1744,7 +1745,7 @@ func testEvalJSONFORGET(t *testing.T, store *dstore.Store) { input: []string{"EXISTING_KEY", "$.*"}, migratedOutput: EvalResponse{ - Result: int64(6), + Result: 6, Error: nil, }, }, @@ -1928,19 +1929,27 @@ func testEvalJSONCLEAR(t *testing.T, store *dstore.Store) { func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { tests := map[string]evalTestCase{ "nil value": { - setup: func() {}, - input: nil, - output: []byte("-ERR wrong number of arguments for 'json.type' command\r\n"), + setup: func() {}, + input: nil, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.TYPE"), + }, }, "empty array": { - setup: func() {}, - input: []string{}, - output: []byte("-ERR wrong number of arguments for 'json.type' command\r\n"), + setup: func() {}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.TYPE"), + }, }, "key does not exist": { - setup: func() {}, - input: []string{"NONEXISTENT_KEY"}, - output: clientio.RespNIL, + setup: func() {}, + input: []string{"NONEXISTENT_KEY"}, + migratedOutput: EvalResponse{ + Result: clientio.NIL, + Error: nil, + }, }, "object type value": { setup: func() { @@ -1952,8 +1961,11 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY"}, - output: []byte("$6\r\nobject\r\n"), + input: []string{"EXISTING_KEY"}, + migratedOutput: EvalResponse{ + Result: "object", + Error: nil, + }, }, "array type value": { setup: func() { @@ -1965,8 +1977,11 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY", "$.language"}, - output: []byte("*1\r\n$5\r\narray\r\n"), + input: []string{"EXISTING_KEY", "$.language"}, + migratedOutput: EvalResponse{ + Result: "array", + Error: nil, + }, }, "string type value": { setup: func() { @@ -1978,8 +1993,11 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY", "$.a"}, - output: []byte("*1\r\n$6\r\nstring\r\n"), + input: []string{"EXISTING_KEY", "$.a"}, + migratedOutput: EvalResponse{ + Result: "string", + Error: nil, + }, }, "boolean type value": { setup: func() { @@ -1991,8 +2009,11 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY", "$.flag"}, - output: []byte("*1\r\n$7\r\nboolean\r\n"), + input: []string{"EXISTING_KEY", "$.flag"}, + migratedOutput: EvalResponse{ + Result: "boolean", + Error: nil, + }, }, "number type value": { setup: func() { @@ -2004,8 +2025,11 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY", "$.price"}, - output: []byte("*1\r\n$6\r\nnumber\r\n"), + input: []string{"EXISTING_KEY", "$.price"}, + migratedOutput: EvalResponse{ + Result: "number", + Error: nil, + }, }, "null type value": { setup: func() { @@ -2017,8 +2041,11 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY", "$.language"}, - output: clientio.RespEmptyArray, + input: []string{"EXISTING_KEY", "$.language"}, + migratedOutput: EvalResponse{ + Result: clientio.EmptyArray, + Error: nil, + }, }, "multi type value": { setup: func() { @@ -2030,30 +2057,42 @@ func testEvalJSONTYPE(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY", "$..name"}, - output: []byte("*2\r\n$6\r\nstring\r\n$6\r\nstring\r\n"), + input: []string{"EXISTING_KEY", "$..name"}, + migratedOutput: EvalResponse{ + Result: "string", + Error: nil, + }, }, } - runEvalTests(t, tests, evalJSONTYPE, store) + runMigratedEvalTests(t, tests, evalJSONTYPE, store) } func testEvalJSONGET(t *testing.T, store *dstore.Store) { tests := map[string]evalTestCase{ "nil value": { - setup: func() {}, - input: nil, - output: []byte("-ERR wrong number of arguments for 'json.get' command\r\n"), + setup: func() {}, + input: nil, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.GET"), + }, }, "empty array": { - setup: func() {}, - input: []string{}, - output: []byte("-ERR wrong number of arguments for 'json.get' command\r\n"), + setup: func() {}, + input: []string{}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.GET"), + }, }, "key does not exist": { - setup: func() {}, - input: []string{"NONEXISTENT_KEY"}, - output: clientio.RespNIL, + setup: func() {}, + input: []string{"NONEXISTENT_KEY"}, + migratedOutput: EvalResponse{ + Result: clientio.NIL, + Error: nil, + }, }, "key exists invalid value": { setup: func() { @@ -2065,8 +2104,11 @@ func testEvalJSONGET(t *testing.T, store *dstore.Store) { } store.Put(key, obj) }, - input: []string{"EXISTING_KEY"}, - output: []byte("-ERR Existing key has wrong Dice type\r\n"), + input: []string{"EXISTING_KEY"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + }, }, "key exists value": { setup: func() { @@ -2078,8 +2120,11 @@ func testEvalJSONGET(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, - input: []string{"EXISTING_KEY"}, - output: []byte("$7\r\n{\"a\":2}\r\n"), + input: []string{"EXISTING_KEY"}, + migratedOutput: EvalResponse{ + Result: "{\"a\":2}", + Error: nil, + }, }, "key exists but expired": { setup: func() { @@ -2093,49 +2138,63 @@ func testEvalJSONGET(t *testing.T, store *dstore.Store) { store.SetExpiry(obj, int64(-2*time.Millisecond)) }, - input: []string{"EXISTING_KEY"}, - output: clientio.RespNIL, + input: []string{"EXISTING_KEY"}, + migratedOutput: EvalResponse{ + Result: clientio.NIL, + Error: nil, + }, }, } - runEvalTests(t, tests, evalJSONGET, store) + runMigratedEvalTests(t, tests, evalJSONGET, store) } func testEvalJSONSET(t *testing.T, store *dstore.Store) { tests := map[string]evalTestCase{ "nil value": { - setup: func() {}, - input: nil, - output: []byte("-ERR wrong number of arguments for 'json.set' command\r\n"), + setup: func() {}, + input: nil, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.SET"), + }, }, "empty array": { - setup: func() {}, - input: []string{}, - output: []byte("-ERR wrong number of arguments for 'json.set' command\r\n"), + setup: func() {}, + input: []string{}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.SET"), + }, }, "insufficient args": { - setup: func() {}, - input: []string{}, - output: []byte("-ERR wrong number of arguments for 'json.set' command\r\n"), + setup: func() {}, + input: []string{}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.SET"), + }, }, "invalid json path": { - setup: func() {}, - input: []string{"doc", "$", "{\"a\":}"}, - output: nil, - validator: func(output []byte) { - assert.True(t, output != nil) - assert.True(t, strings.Contains(string(output), "-ERR invalid JSON:")) + setup: func() {}, + input: []string{"doc", "$", "{\"a\":}"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid JSON"), }, }, "valid json path": { setup: func() { }, - input: []string{"doc", "$", "{\"a\":2}"}, - output: clientio.RespOK, + input: []string{"doc", "$", "{\"a\":2}"}, + migratedOutput: EvalResponse{ + Result: clientio.OK, + Error: nil, + }, }, } - runEvalTests(t, tests, evalJSONSET, store) + runMigratedEvalTests(t, tests, evalJSONSET, store) } func testEvalJSONNUMMULTBY(t *testing.T, store *dstore.Store) { @@ -3159,6 +3218,70 @@ func testEvalHGET(t *testing.T, store *dstore.Store) { runMigratedEvalTests(t, tests, evalHGET, store) } +func testEvalHGETALL(t *testing.T, store *dstore.Store) { + tests := map[string]evalTestCase{ + "wrong number of args passed": { + setup: func() {}, + input: []string{}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("HGETALL"), + }, + }, + "key doesn't exist": { + setup: func() {}, + input: []string{"NON_EXISTENT_KEY"}, + migratedOutput: EvalResponse{ + Result: []string{}, // Expect empty result as the key doesn't exist + Error: nil, + }, + }, + "key exists but is empty": { + setup: func() { + key := "EMPTY_HASH" + newMap := make(HashMap) // Empty hash map + + obj := &object.Obj{ + TypeEncoding: object.ObjTypeHashMap | object.ObjEncodingHashMap, + Value: newMap, + LastAccessedAt: uint32(time.Now().Unix()), + } + + store.Put(key, obj) + }, + input: []string{"EMPTY_HASH"}, + migratedOutput: EvalResponse{ + Result: []string{}, // Expect empty result as the hash is empty + Error: nil, + }, + }, + "key exists with multiple fields": { + setup: func() { + key := "HASH_WITH_FIELDS" + newMap := make(HashMap) + newMap["field1"] = "value1" + newMap["field2"] = "value2" + newMap["field3"] = "value3" + + obj := &object.Obj{ + TypeEncoding: object.ObjTypeHashMap | object.ObjEncodingHashMap, + Value: newMap, + LastAccessedAt: uint32(time.Now().Unix()), + } + + store.Put(key, obj) + }, + input: []string{"HASH_WITH_FIELDS"}, + migratedOutput: EvalResponse{ + Result: []string{"field1", "value1", "field2", "value2", "field3", "value3"}, + Error: nil, + }, + }, + } + + runMigratedEvalTests(t, tests, evalHGETALL, store) +} + func testEvalHMGET(t *testing.T, store *dstore.Store) { tests := map[string]evalTestCase{ "wrong number of args passed": { @@ -4224,6 +4347,8 @@ func runMigratedEvalTests(t *testing.T, tests map[string]evalTestCase, evalFunc // TODO: Make this generic so that all kind of slices can be handled if b, ok := output.Result.([]byte); ok && tc.migratedOutput.Result != nil { if expectedBytes, ok := tc.migratedOutput.Result.([]byte); ok { + fmt.Println(string(b)) + fmt.Println(string(expectedBytes)) assert.True(t, bytes.Equal(b, expectedBytes), "expected and actual byte slices should be equal") } } else if a, ok := output.Result.([]string); ok && tc.migratedOutput.Result != nil { @@ -8141,19 +8266,28 @@ func BenchmarkEvalHINCRBYFLOAT(b *testing.B) { func testEvalDUMP(t *testing.T, store *dstore.Store) { tests := map[string]evalTestCase{ "nil value": { - setup: func() {}, - input: nil, - output: []byte("-ERR wrong number of arguments for 'dump' command\r\n"), + setup: func() {}, + input: nil, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("DUMP"), + }, }, "empty array": { - setup: func() {}, - input: []string{}, - output: []byte("-ERR wrong number of arguments for 'dump' command\r\n"), + setup: func() {}, + input: []string{}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("DUMP"), + }, }, "key does not exist": { - setup: func() {}, - input: []string{"NONEXISTENT_KEY"}, - output: []byte("-ERR nil\r\n"), + setup: func() {}, + input: []string{"NONEXISTENT_KEY"}, + migratedOutput: EvalResponse{ + Result: clientio.NIL, + Error: nil, + }, }, "dump string value": { setup: func() { key := "user" @@ -8162,13 +8296,15 @@ func testEvalDUMP(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, input: []string{"user"}, - output: clientio.Encode( - base64.StdEncoding.EncodeToString([]byte{ + migratedOutput: EvalResponse{ + Result: base64.StdEncoding.EncodeToString([]byte{ 0x09, 0x00, 0x00, 0x00, 0x00, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0xFF, // End marker // CRC64 checksum here: 0x00, 0x47, 0x97, 0x93, 0xBE, 0x36, 0x45, 0xC7, - }), false), + }), + Error: nil, + }, }, "dump integer value": { setup: func() { @@ -8178,13 +8314,16 @@ func testEvalDUMP(t *testing.T, store *dstore.Store) { store.Put(key, obj) }, input: []string{"INTEGER_KEY"}, - output: clientio.Encode(base64.StdEncoding.EncodeToString([]byte{ - 0x09, - 0xC0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, - 0xFF, - 0x12, 0x77, 0xDE, 0x29, 0x53, 0xDB, 0x44, 0xC2, - }), false), + migratedOutput: EvalResponse{ + Result: base64.StdEncoding.EncodeToString([]byte{ + 0x09, + 0xC0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, + 0xFF, + 0x12, 0x77, 0xDE, 0x29, 0x53, 0xDB, 0x44, 0xC2, + }), + Error: nil, + }, }, "dump expired key": { setup: func() { @@ -8195,12 +8334,15 @@ func testEvalDUMP(t *testing.T, store *dstore.Store) { var exDurationMs int64 = -1 store.SetExpiry(obj, exDurationMs) }, - input: []string{"EXPIRED_KEY"}, - output: []byte("-ERR nil\r\n"), + input: []string{"EXPIRED_KEY"}, + migratedOutput: EvalResponse{ + Result: clientio.NIL, + Error: nil, + }, }, } - runEvalTests(t, tests, evalDUMP, store) + runMigratedEvalTests(t, tests, evalDUMP, store) } func testEvalBitFieldRO(t *testing.T, store *dstore.Store) { @@ -8438,15 +8580,21 @@ func testEvalJSONSTRAPPEND(t *testing.T, store *dstore.Store) { obj := store.NewObj(rootData, -1, object.ObjTypeJSON, object.ObjEncodingJSON) store.Put(key, obj) }, - input: []string{"doc1", "$.nested1.a", "\"baz\""}, - output: []byte("*1\r\n:8\r\n"), // Expected length after append + input: []string{"doc1", "$.nested1.a", "\"baz\""}, + migratedOutput: EvalResponse{ + Result: 8, + Error: nil, + }, }, "append to non-existing key": { setup: func() { // No setup needed as we are testing a non-existing document. }, - input: []string{"non_existing_doc", "$..a", "\"err\""}, - output: []byte("-ERR Could not perform this operation on a key that doesn't exist\r\n"), + input: []string{"non_existing_doc", "$..a", "\"err\""}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrKeyDoesNotExist, + }, }, "append to root node": { setup: func() { @@ -8457,13 +8605,16 @@ func testEvalJSONSTRAPPEND(t *testing.T, store *dstore.Store) { obj := store.NewObj(rootData, -1, object.ObjTypeJSON, object.ObjEncodingJSON) store.Put(key, obj) }, - input: []string{"doc1", "$", "\"piu\""}, - output: []byte("*1\r\n:7\r\n"), // Expected length after appending to "abcd" + input: []string{"doc1", "$", "\"piu\""}, + migratedOutput: EvalResponse{ + Result: 7, + Error: nil, + }, }, } // Run the tests - runEvalTests(t, tests, evalJSONSTRAPPEND, store) + runMigratedEvalTests(t, tests, evalJSONSTRAPPEND, store) } func BenchmarkEvalJSONSTRAPPEND(b *testing.B) { diff --git a/internal/eval/execute.go b/internal/eval/execute.go index 35cdda04e..ee159ad18 100644 --- a/internal/eval/execute.go +++ b/internal/eval/execute.go @@ -11,18 +11,58 @@ import ( dstore "github.com/dicedb/dice/internal/store" ) -func ExecuteCommand(c *cmd.DiceDBCmd, client *comm.Client, store *dstore.Store, httpOp, websocketOp bool) *EvalResponse { - diceCmd, ok := DiceCmds[c.Cmd] +type Eval struct { + cmd *cmd.DiceDBCmd + client *comm.Client + store *dstore.Store + isHTTPOperation bool + isWebSocketOperation bool + isPreprocessOperation bool +} + +func NewEval(c *cmd.DiceDBCmd, client *comm.Client, store *dstore.Store, httpOp, websocketOp, preProcessing bool) *Eval { + return &Eval{ + cmd: c, + client: client, + store: store, + isHTTPOperation: httpOp, + isWebSocketOperation: websocketOp, + isPreprocessOperation: preProcessing, + } +} + +func (e *Eval) PreProcessCommand() *EvalResponse { + if f, ok := PreProcessing[e.cmd.Cmd]; ok { + return f(e.cmd.Args, e.store) + } + return &EvalResponse{Result: nil, Error: diceerrors.ErrInternalServer} +} + +func (e *Eval) ExecuteCommand() *EvalResponse { + diceCmd, ok := DiceCmds[e.cmd.Cmd] if !ok { - return &EvalResponse{Result: diceerrors.NewErrWithFormattedMessage("unknown command '%s', with args beginning with: %s", c.Cmd, strings.Join(c.Args, " ")), Error: nil} + return &EvalResponse{Result: diceerrors.NewErrWithFormattedMessage("unknown command '%s', with args beginning with: %s", e.cmd.Cmd, strings.Join(e.cmd.Args, " ")), Error: nil} } // Temporary logic till we move all commands to new eval logic. // MigratedDiceCmds map contains refactored eval commands // For any command we will first check in the existing map // if command is NA then we will check in the new map + // Check if the dice command has been migrated if diceCmd.IsMigrated { - return diceCmd.NewEval(c.Args, store) + // =============================================================================== + // dealing with store object is not recommended for all commands + // These operations are specialised for the commands which requires + // transfering data across multiple shards. e.g COPY, RENAME + // =============================================================================== + if e.cmd.InternalObj != nil { + // This involves handling object at store level, evaluating it, modifying it, and then storing it back. + return diceCmd.StoreObjectEval(e.cmd, e.store) + } + + // If the 'Obj' field is nil, handle the command using the arguments. + // This path likely involves evaluating the command based on its provided arguments. + return diceCmd.NewEval(e.cmd.Args, e.store) } // The following commands could be handled at the shard level, however, we can randomly let any shard handle them @@ -31,14 +71,14 @@ func ExecuteCommand(c *cmd.DiceDBCmd, client *comm.Client, store *dstore.Store, // Old implementation kept as it is, but we will be moving // to the new implementation soon for all commands case "SUBSCRIBE", "Q.WATCH": - return &EvalResponse{Result: EvalQWATCH(c.Args, httpOp, websocketOp, client, store), Error: nil} + return &EvalResponse{Result: EvalQWATCH(e.cmd.Args, e.isHTTPOperation, e.isWebSocketOperation, e.client, e.store), Error: nil} case "UNSUBSCRIBE", "Q.UNWATCH": - return &EvalResponse{Result: EvalQUNWATCH(c.Args, httpOp, client), Error: nil} + return &EvalResponse{Result: EvalQUNWATCH(e.cmd.Args, e.isHTTPOperation, e.client), Error: nil} case auth.Cmd: - return &EvalResponse{Result: EvalAUTH(c.Args, client), Error: nil} + return &EvalResponse{Result: EvalAUTH(e.cmd.Args, e.client), Error: nil} case "ABORT": return &EvalResponse{Result: clientio.RespOK, Error: nil} default: - return &EvalResponse{Result: diceCmd.Eval(c.Args, store), Error: nil} + return &EvalResponse{Result: diceCmd.Eval(e.cmd.Args, e.store), Error: nil} } } diff --git a/internal/eval/store_eval.go b/internal/eval/store_eval.go index eaa751db1..cb675edec 100644 --- a/internal/eval/store_eval.go +++ b/internal/eval/store_eval.go @@ -1,6 +1,7 @@ package eval import ( + "encoding/base64" "errors" "fmt" "math" @@ -15,6 +16,7 @@ import ( "github.com/axiomhq/hyperloglog" "github.com/bytedance/sonic" "github.com/dicedb/dice/internal/clientio" + "github.com/dicedb/dice/internal/cmd" diceerrors "github.com/dicedb/dice/internal/errors" "github.com/dicedb/dice/internal/eval/sortedset" "github.com/dicedb/dice/internal/object" @@ -22,6 +24,7 @@ import ( dstore "github.com/dicedb/dice/internal/store" "github.com/gobwas/glob" "github.com/ohler55/ojg/jp" + "github.com/rs/xid" ) // evalEXPIRE sets an expiry time(in secs) on the specified key in args @@ -1351,6 +1354,350 @@ func evalJSONCLEAR(args []string, store *dstore.Store) *EvalResponse { } } +// evalJSONGET retrieves a JSON value stored at the specified key +// args must contain at least the key; (path unused in this implementation) +// Returns response.RespNIL if key is expired, or it does not exist +// Returns encoded error response if incorrect number of arguments +// The RESP value of the key is encoded and then returned +func evalJSONGET(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 1 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.GET"), + } + } + + key := args[0] + // Default path is root if not specified + path := defaultRootPath + if len(args) > 1 { + path = args[1] + } + return jsonGETHelper(store, path, key) +} + +// helper function used by evalJSONGET and evalJSONMGET to prepare the results +func jsonGETHelper(store *dstore.Store, path, key string) *EvalResponse { + // Retrieve the object from the database + obj := store.Get(key) + if obj == nil { + return &EvalResponse{ + Result: clientio.NIL, + Error: nil, + } + } + + // Check if the object is of JSON type + errWithMessage := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeJSON, object.ObjEncodingJSON) + if errWithMessage != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + + jsonData := obj.Value + + // If path is root, return the entire JSON + if path == defaultRootPath { + resultBytes, err := sonic.Marshal(jsonData) + fmt.Println(string(resultBytes)) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("could not serialize result"), + } + } + + return &EvalResponse{ + Result: string(resultBytes), + Error: nil, + } + } + + // Parse the JSONPath expression + expr, err := jp.ParseString(path) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid JSONPath"), + } + } + + // Execute the JSONPath query + results := expr.Get(jsonData) + if len(results) == 0 { + return &EvalResponse{ + Result: clientio.NIL, + Error: nil, + } + } + + // Serialize the result + var resultBytes []byte + if len(results) == 1 { + resultBytes, err = sonic.Marshal(results[0]) + } else { + resultBytes, err = sonic.Marshal(results) + } + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("could not serialize result"), + } + } + return &EvalResponse{ + Result: string(resultBytes), + Error: nil, + } +} + +// evalJSONSET stores a JSON value at the specified key +// args must contain at least the key, path (unused in this implementation), and JSON string +// Returns encoded error response if incorrect number of arguments +// Returns encoded error if the JSON string is invalid +// Returns response.RespOK if the JSON value is successfully stored +func evalJSONSET(args []string, store *dstore.Store) *EvalResponse { + // Check if there are enough arguments + if len(args) < 3 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.SET"), + } + } + + key := args[0] + path := args[1] + jsonStr := args[2] + for i := 3; i < len(args); i++ { + switch strings.ToUpper(args[i]) { + case NX: + if i != len(args)-1 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrSyntax, + } + } + obj := store.Get(key) + if obj != nil { + return &EvalResponse{ + Result: clientio.NIL, + Error: nil, + } + } + case XX: + if i != len(args)-1 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrSyntax, + } + } + obj := store.Get(key) + if obj == nil { + return &EvalResponse{ + Result: clientio.NIL, + Error: nil, + } + } + + default: + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrSyntax, + } + } + } + + // Parse the JSON string + var jsonValue interface{} + if err := sonic.UnmarshalString(jsonStr, &jsonValue); err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid JSON"), + } + } + + // Retrieve existing object or create new one + obj := store.Get(key) + var rootData interface{} + + if obj == nil { + // If the key doesn't exist, create a new object + if path != defaultRootPath { + rootData = make(map[string]interface{}) + } else { + rootData = jsonValue + } + } else { + // If the key exists, check if it's a JSON object + err := object.AssertType(obj.TypeEncoding, object.ObjTypeJSON) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + err = object.AssertEncoding(obj.TypeEncoding, object.ObjEncodingJSON) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + rootData = obj.Value + } + + // If path is not root, use JSONPath to set the value + if path != defaultRootPath { + expr, err := jp.ParseString(path) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid JSONPath"), + } + } + + err = expr.Set(rootData, jsonValue) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("failed to set value"), + } + } + } else { + // If path is root, replace the entire JSON + rootData = jsonValue + } + + // Create a new object with the updated JSON data + newObj := store.NewObj(rootData, -1, object.ObjTypeJSON, object.ObjEncodingJSON) + store.Put(key, newObj) + return &EvalResponse{ + Result: clientio.OK, + Error: nil, + } +} + +// evalJSONINGEST stores a value at a dynamically generated key +// The key is created using a provided key prefix combined with a unique identifier +// args must contains key_prefix and path and json value +// It will call to evalJSONSET internally. +// Returns encoded error response if incorrect number of arguments +// Returns encoded error if the JSON string is invalid +// Returns unique identifier if the JSON value is successfully stored +func evalJSONINGEST(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 3 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.INGEST"), + } + } + + keyPrefix := args[0] + + uniqueID := xid.New() + uniqueKey := keyPrefix + uniqueID.String() + + var setArgs []string + setArgs = append(setArgs, uniqueKey) + setArgs = append(setArgs, args[1:]...) + + result := evalJSONSET(setArgs, store) + if resultValue, ok := result.Result.(clientio.RespType); ok { + // If Result is of type RespType, check equality + if resultValue == clientio.OK { + return &EvalResponse{ + Result: uniqueID.String(), + Error: nil, + } + } + } + return result +} + +// evalJSONTYPE retrieves a JSON value type stored at the specified key +// args must contain at least the key; (path unused in this implementation) +// Returns response.RespNIL if key is expired, or it does not exist +// Returns encoded error response if incorrect number of arguments +// The RESP value of the key's value type is encoded and then returned +func evalJSONTYPE(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 1 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("JSON.TYPE"), + } + } + key := args[0] + + // Default path is root if not specified + path := defaultRootPath + if len(args) > 1 { + path = args[1] + } + // Retrieve the object from the database + obj := store.Get(key) + if obj == nil { + return &EvalResponse{ + Result: clientio.NIL, + Error: nil, + } + } + + errWithMessage := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeJSON, object.ObjEncodingJSON) + if errWithMessage != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + + jsonData := obj.Value + + if path == defaultRootPath { + _, err := sonic.Marshal(jsonData) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("could not serialize result"), + } + } + // If path is root and len(args) == 1, return "object" instantly + if len(args) == 1 { + return &EvalResponse{ + Result: utils.ObjectType, + Error: nil, + } + } + } + + // Parse the JSONPath expression + expr, err := jp.ParseString(path) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid JSONPath"), + } + } + + results := expr.Get(jsonData) + if len(results) == 0 { + return &EvalResponse{ + Result: clientio.EmptyArray, + Error: nil, + } + } + + typeList := make([]string, 0, len(results)) + for _, result := range results { + jsonType := utils.GetJSONFieldType(result) + typeList = append(typeList, jsonType) + } + return &EvalResponse{ + Result: typeList, + Error: nil, + } +} + // PFADD Adds all the element arguments to the HyperLogLog data structure stored at the variable // name specified as first argument. // @@ -2292,6 +2639,57 @@ func evalZPOPMIN(args []string, store *dstore.Store) *EvalResponse { } } +func evalDUMP(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 1 { + return makeEvalError(diceerrors.ErrWrongArgumentCount("DUMP")) + } + key := args[0] + obj := store.Get(key) + if obj == nil { + return makeEvalResult(clientio.NIL) + } + + serializedValue, err := rdbSerialize(obj) + if err != nil { + return makeEvalError(diceerrors.ErrGeneral("serialization failed")) + } + encodedResult := base64.StdEncoding.EncodeToString(serializedValue) + + return makeEvalResult(encodedResult) +} + +func evalRestore(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 3 { + return makeEvalError(diceerrors.ErrWrongArgumentCount("RESTORE")) + } + + key := args[0] + ttlStr := args[1] + ttl, _ := strconv.ParseInt(ttlStr, 10, 64) + + encodedValue := args[2] + serializedData, err := base64.StdEncoding.DecodeString(encodedValue) + if err != nil { + return makeEvalError(diceerrors.ErrGeneral("failed to decode base64 value")) + } + + obj, err := rdbDeserialize(serializedData) + if err != nil { + return makeEvalError(diceerrors.ErrGeneral("deserialization failed")) + } + + newobj := store.NewObj(obj.Value, ttl, obj.TypeEncoding, obj.TypeEncoding) + var keepttl = true + + if ttl > 0 { + store.Put(key, newobj, dstore.WithKeepTTL(keepttl)) + } else { + store.Put(key, obj) + } + + return makeEvalResult(clientio.OK) +} + // evalHLEN returns the number of fields contained in the hash stored at key. // // If key doesn't exist, it returns 0. @@ -4331,6 +4729,41 @@ func evalHGET(args []string, store *dstore.Store) *EvalResponse { } } +func evalHGETALL(args []string, store *dstore.Store) *EvalResponse { + if len(args) != 1 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("HGETALL"), + } + } + + key := args[0] + + obj := store.Get(key) + + var hashMap HashMap + var results []string + + if obj != nil { + if err := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeHashMap, object.ObjEncodingHashMap); err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + hashMap = obj.Value.(HashMap) + } + + for hmKey, hmValue := range hashMap { + results = append(results, hmKey, hmValue) + } + + return &EvalResponse{ + Result: results, + Error: nil, + } +} + func evalHSETNX(args []string, store *dstore.Store) *EvalResponse { if len(args) != 3 { return &EvalResponse{ @@ -5158,6 +5591,115 @@ func evalBITFIELDRO(args []string, store *dstore.Store) *EvalResponse { return bitfieldEvalGeneric(args, store, true) } +func evalGetObject(args []string, store *dstore.Store) *EvalResponse { + if len(args) != 1 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrInternalServer, + } + } + + key := args[0] + + obj := store.Get(key) + + // if key does not exist, return RESP encoded nil + if obj == nil { + return &EvalResponse{ + Result: clientio.IntegerZero, + Error: nil, + } + } + + exp, ok := dstore.GetExpiry(obj, store) + var exDurationMs int64 = -1 + if ok { + exDurationMs = int64(exp - uint64(utils.GetCurrentTime().UnixMilli())) + } + + exObj := &object.InternalObj{ + Obj: obj, + ExDuration: exDurationMs, + } + + // Decode and return the value based on its encoding + return &EvalResponse{ + Result: exObj, + Error: nil, + } +} + +// evalJSONSTRAPPEND appends a string value to the JSON string value at the specified path +// in the JSON object saved at the key in arguments. +// Args must contain at least a key and the string value to append. +// If the key does not exist or is expired, it returns an error response. +// If the value at the specified path is not a string, it returns an error response. +// Returns the new length of the string at the specified path if successful. +func evalJSONSTRAPPEND(args []string, store *dstore.Store) *EvalResponse { + if len(args) != 3 { + return makeEvalError(diceerrors.ErrWrongArgumentCount("JSON.STRAPPEND")) + } + + key := args[0] + path := args[1] + value := args[2] + + obj := store.Get(key) + if obj == nil { + return makeEvalError(diceerrors.ErrKeyDoesNotExist) + } + + errWithMessage := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeJSON, object.ObjEncodingJSON) + if errWithMessage != nil { + return makeEvalError(diceerrors.ErrWrongTypeOperation) + } + + jsonData := obj.Value + + var resultsArray []interface{} + + if path == "$" { + // Handle root-level string + if str, ok := jsonData.(string); ok { + unquotedValue := strings.Trim(value, "\"") + newValue := str + unquotedValue + resultsArray = append(resultsArray, len(newValue)) + jsonData = newValue + } else { + return makeEvalResult(clientio.EmptyArray) + } + } else { + expr, err := jp.ParseString(path) + if err != nil { + return makeEvalResult(clientio.EmptyArray) + } + + _, modifyErr := expr.Modify(jsonData, func(data any) (interface{}, bool) { + switch v := data.(type) { + case string: + unquotedValue := strings.Trim(value, "\"") + newValue := v + unquotedValue + resultsArray = append([]interface{}{len(newValue)}, resultsArray...) + return newValue, true + default: + resultsArray = append([]interface{}{clientio.RespNIL}, resultsArray...) + return data, false + } + }) + + if modifyErr != nil { + return makeEvalResult(clientio.EmptyArray) + } + } + + if len(resultsArray) == 0 { + return makeEvalResult(clientio.EmptyArray) + } + + obj.Value = jsonData + return makeEvalResult(resultsArray[0]) +} + // evalJSONTOGGLE toggles a boolean value stored at the specified key and path. // args must contain at least the key and path (where the boolean is located). // If the key does not exist or is expired, it returns response.RespNIL. @@ -5262,7 +5804,7 @@ func evalJSONDEL(args []string, store *dstore.Store) *EvalResponse { // Retrieve the object from the database obj := store.Get(key) if obj == nil { - return makeEvalResult(0) + return makeEvalResult(clientio.IntegerZero) } errWithMessage := object.AssertTypeAndEncoding(obj.TypeEncoding, object.ObjTypeJSON, object.ObjEncodingJSON) @@ -5279,7 +5821,7 @@ func evalJSONDEL(args []string, store *dstore.Store) *EvalResponse { if len(args) == 1 || path == defaultRootPath { store.Del(key) - return makeEvalResult(int64(1)) + return makeEvalResult(1) } expr, err := jp.ParseString(path) @@ -5298,13 +5840,13 @@ func evalJSONDEL(args []string, store *dstore.Store) *EvalResponse { } if err != nil { - return makeEvalError(diceerrors.ErrGeneral(err.Error())) + return makeEvalError(diceerrors.ErrInternalServer) // no need to send actual internal error } // Create a new object with the updated JSON data newObj := store.NewObj(jsonData, -1, object.ObjTypeJSON, object.ObjEncodingJSON) store.Put(key, newObj) - return makeEvalResult(int64(len(results))) + return makeEvalResult(len(results)) } // Returns the new value after incrementing or multiplying the existing value @@ -5456,6 +5998,54 @@ func evalJSONNUMMULTBY(args []string, store *dstore.Store) *EvalResponse { return makeEvalResult(resultString) } +// evalSetObject stores an object in the store with a given key and optional expiry. +// If an object with the same key exists, it is replaced. +// This function is usually specifc to multishard multi-op commands +func evalCOPYObject(cd *cmd.DiceDBCmd, store *dstore.Store) *EvalResponse { + args := cd.Args + + var isReplace bool + if len(cd.Args) > 1 { + if cd.Args[1] == "REPLACE" { + isReplace = true + } + } + + key := args[0] + + obj := store.Get(key) + + if !isReplace && obj != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("ERR target key already exists"), + } + } + + store.Del(key) + + copyObj := cd.InternalObj.Obj.DeepCopy() + if copyObj == nil { + return &EvalResponse{ + Result: clientio.IntegerZero, + Error: nil, + } + } + + exDurationMs := cd.InternalObj.ExDuration + + store.Put(key, copyObj) + + if exDurationMs > 0 { + store.SetExpiry(copyObj, exDurationMs) + } + + return &EvalResponse{ + Result: clientio.IntegerOne, + Error: nil, + } +} + // takes original value, increment values (float or int), a flag representing if increment is float // returns new value, string representation, a boolean representing if the value was modified func incrementValue(value any, isIncrFloat bool, incrFloat float64, incrInt int64) (newVal interface{}, stringRepresentation string, isModified bool) { diff --git a/internal/object/object.go b/internal/object/object.go index fdc736e8c..797a0f518 100644 --- a/internal/object/object.go +++ b/internal/object/object.go @@ -1,14 +1,75 @@ package object +// Obj represents a basic object structure that includes metadata about the object +// as well as its value. This struct is designed to store the core data (value) +// of the object along with additional properties such as encoding type and access +// time. The structure is intended to support different types of objects, including +// simple types (e.g., int, string) and more complex data structures (e.g., hashes, sets). +// +// Fields: +// +// - TypeEncoding: A uint8 field used to store the encoding type of the object. This +// helps in identifying how the object is encoded or serialized (e.g., as a string, +// number, or more complex type). It is crucial for determining how the object should +// be interpreted or processed when retrieved from storage. +// +// - LastAccessedAt: A uint32 field that stores the timestamp (in seconds or milliseconds) +// representing the last time the object was accessed. This field helps in tracking the +// freshness of the object and can be used for cache expiry or eviction policies. +// in DiceDB we use 32 bits due to Go's lack of native support for bitfields, +// and to simplify management by not combining +// `TypeEncoding` and `LastAccessedAt` into a single integer. +// +// - Value: An `interface{}` type that holds the actual data of the object. This could +// represent any type of data, allowing flexibility to store different kinds of +// objects (e.g., strings, numbers, complex data structures like lists or maps). type Obj struct { + // TypeEncoding holds the encoding type of the object (e.g., string, int, complex structure) TypeEncoding uint8 - // Redis allocates 24 bits to these bits, but we will use 32 bits because - // golang does not support bitfields, and we need not make this super-complicated - // by merging TypeEncoding + LastAccessedAt in one 32-bit integer. - // But nonetheless, we can benchmark and see how that fares. - // For now, we continue with 32-bit integer to Store the LastAccessedAt + + // LastAccessedAt stores the last access timestamp of the object. + // It helps track when the object was last accessed and may be used for cache eviction or freshness tracking. LastAccessedAt uint32 - Value interface{} + + // Value holds the actual content or data of the object, which can be of any type. + // This allows flexibility in storing various kinds of objects (simple or complex). + Value interface{} +} + +// ExtendedObj is an extension of the `Obj` struct, designed to add extra +// metadata for more advanced use cases. It includes a reference to an `Obj` +// and an additional field `ExDuration` to store a time-based property (e.g., +// the duration the object is valid or how long it should be stored). +// +// This struct allows for greater flexibility in managing objects that have +// additional time-sensitive or context-dependent attributes. The `ExDuration` +// field can represent the lifespan, expiration time, or any other time-related +// characteristic associated with the object. +// +// **Caution**: `InternalObj` should be used selectively and with caution. +// It is intended for commands that require the additional metadata provided by +// the `ExDuration` field. Using this for all objects can complicate logic and +// may lead to unnecessary overhead. It is recommended to use `ExtendedObj` only +// for commands that specifically require it and when additional metadata is +// necessary for the functionality of the command. +// +// Fields: +// +// - Obj: A pointer to the underlying `Obj` that contains the core object data +// (such as encoding type, last accessed timestamp, and the actual value of the object). +// This allows the extended object to retain all of the functionality of the basic object +// while adding more flexibility through the extra metadata. +// +// - ExDuration: Represents a time-based property, such as the duration for which +// the object is valid or the time it should be stored or processed. This allows for +// managing the object with respect to time-based characteristics, such as expiration. +type InternalObj struct { + // Obj holds the core object structure with the basic metadata and value + Obj *Obj + + // ExDuration represents a time-based property, such as the duration for which + // the object is valid or the time it should be stored or processed. + ExDuration int64 } var ObjTypeString uint8 = 0 << 4 diff --git a/internal/server/cmd_meta.go b/internal/server/cmd_meta.go index 995b24def..5f0677d7e 100644 --- a/internal/server/cmd_meta.go +++ b/internal/server/cmd_meta.go @@ -123,6 +123,7 @@ var ( Cmd: "HKEYS", CmdType: SingleShard, } + hvalsCmdMeta = CmdsMeta{ Cmd: "HVALS", CmdType: SingleShard, @@ -402,7 +403,38 @@ var ( Cmd: "JSON.NUMMULTBY", CmdType: SingleShard, } - + jsonSetCmdMeta = CmdsMeta{ + Cmd: "JSON.SET", + CmdType: SingleShard, + } + jsonGetCmdMeta = CmdsMeta{ + Cmd: "JSON.GET", + CmdType: SingleShard, + } + jsonTypeCmdMeta = CmdsMeta{ + Cmd: "JSON.TYPE", + CmdType: SingleShard, + } + jsonIngestCmdMeta = CmdsMeta{ + Cmd: "JSON.INGEST", + CmdType: SingleShard, + } + jsonArrStrAppendCmdMeta = CmdsMeta{ + Cmd: "JSON.STRAPPEND", + CmdType: SingleShard, + } + hGetAllCmdMeta = CmdsMeta{ + Cmd: "HGETALL", + CmdType: SingleShard, + } + dumpCmdMeta = CmdsMeta{ + Cmd: "DUMP", + CmdType: SingleShard, + } + restoreCmdMeta = CmdsMeta{ + Cmd: "RESTORE", + CmdType: SingleShard, + } // Metadata for multishard commands would go here. // These commands require both breakup and gather logic. @@ -519,5 +551,13 @@ func init() { WorkerCmdsMeta["JSON.TOGGLE"] = jsonToggleCmdMeta WorkerCmdsMeta["JSON.NUMINCRBY"] = jsonNumIncrByCmdMeta WorkerCmdsMeta["JSON.NUMMULTBY"] = jsonNumMultByCmdMeta + WorkerCmdsMeta["JSON.SET"] = jsonSetCmdMeta + WorkerCmdsMeta["JSON.GET"] = jsonGetCmdMeta + WorkerCmdsMeta["JSON.TYPE"] = jsonTypeCmdMeta + WorkerCmdsMeta["JSON.INGEST"] = jsonIngestCmdMeta + WorkerCmdsMeta["JSON.STRAPPEND"] = jsonArrStrAppendCmdMeta + WorkerCmdsMeta["HGETALL"] = hGetAllCmdMeta + WorkerCmdsMeta["DUMP"] = dumpCmdMeta + WorkerCmdsMeta["RESTORE"] = restoreCmdMeta // Additional commands (multishard, custom) can be added here as needed. } diff --git a/internal/server/server.go b/internal/server/server.go index 3b27efa79..7ed068d34 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -421,9 +421,6 @@ func (s *AsyncServer) handleTransactionCommand(diceDBCmd *cmd.DiceDBCmd, c *comm func (s *AsyncServer) handleNonTransactionCommand(diceDBCmd *cmd.DiceDBCmd, c *comm.Client, buf *bytes.Buffer) { switch diceDBCmd.Cmd { - case eval.MultiCmdMeta.Name: - c.TxnBegin() - buf.Write(clientio.RespOK) case eval.ExecCmdMeta.Name: buf.Write(diceerrors.NewErrWithMessage("EXEC without MULTI")) case eval.DiscardCmdMeta.Name: diff --git a/internal/shard/shard_thread.go b/internal/shard/shard_thread.go index 1e7b5be26..a556482e0 100644 --- a/internal/shard/shard_thread.go +++ b/internal/shard/shard_thread.go @@ -96,8 +96,6 @@ func (shard *ShardThread) unregisterWorker(workerID string) { // processRequest processes a Store operation for the shard. func (shard *ShardThread) processRequest(op *ops.StoreOp) { - resp := eval.ExecuteCommand(op.Cmd, op.Client, shard.store, op.HTTPOp, op.WebsocketOp) - shard.workerMutex.RLock() workerChans, ok := shard.workerMap[op.WorkerID] shard.workerMutex.RUnlock() @@ -110,6 +108,16 @@ func (shard *ShardThread) processRequest(op *ops.StoreOp) { SeqID: op.SeqID, } + e := eval.NewEval(op.Cmd, op.Client, shard.store, op.HTTPOp, op.WebsocketOp, op.PreProcessing) + + if op.PreProcessing { + resp := e.PreProcessCommand() + sp.EvalResponse = resp + preProcessChan <- sp + return + } + + resp := e.ExecuteCommand() if ok { sp.EvalResponse = resp } else { @@ -119,11 +127,7 @@ func (shard *ShardThread) processRequest(op *ops.StoreOp) { } } - if op.PreProcessing { - preProcessChan <- sp - } else { - workerChan <- sp - } + workerChan <- sp } // cleanup handles cleanup logic when the shard stops. diff --git a/internal/store/constants.go b/internal/store/constants.go index 53deb0050..fad990f1c 100644 --- a/internal/store/constants.go +++ b/internal/store/constants.go @@ -1,13 +1,17 @@ package store const ( - Set string = "SET" - Del string = "DEL" - Get string = "GET" - Rename string = "RENAME" - ZAdd string = "ZADD" - ZRange string = "ZRANGE" - PFADD string = "PFADD" - PFCOUNT string = "PFCOUNT" - PFMERGE string = "PFMERGE" + Set string = "SET" + Del string = "DEL" + Get string = "GET" + Rename string = "RENAME" + ZAdd string = "ZADD" + ZRange string = "ZRANGE" + Replace string = "REPLACE" + Smembers string = "SMEMBERS" + JSONGet string = "JSON.GET" + PFADD string = "PFADD" + PFCOUNT string = "PFCOUNT" + PFMERGE string = "PFMERGE" + KEYSPERSHARD string = "KEYSPERSHARD" ) diff --git a/internal/worker/cmd_compose.go b/internal/worker/cmd_compose.go index 5a46c4986..cb5c291c8 100644 --- a/internal/worker/cmd_compose.go +++ b/internal/worker/cmd_compose.go @@ -1,6 +1,7 @@ package worker import ( + "math" "sort" "github.com/dicedb/dice/internal/clientio" @@ -44,7 +45,7 @@ func composeCopy(responses ...ops.StoreResponse) interface{} { } } - return clientio.OK + return clientio.IntegerOne } // composeMSet processes responses from multiple shards for an "MSet" operation @@ -83,3 +84,112 @@ func composeMGet(responses ...ops.StoreResponse) interface{} { return results } + +func composeSInter(responses ...ops.StoreResponse) interface{} { + results := [][]string{} + minLen := math.MaxInt + minIdx := 0 + for idx := range responses { + if responses[idx].EvalResponse.Error != nil { + return responses[idx].EvalResponse.Error + } + + if len(responses[idx].EvalResponse.Result.([]string)) < minLen { + minLen = len(responses[idx].EvalResponse.Result.([]string)) + minIdx = idx + } + + results = append(results, responses[idx].EvalResponse.Result.([]string)) + } + + // Create the initial countMap from the smallest slice + countMap := make(map[string]int) + for _, str := range results[minIdx] { + countMap[str] = 1 + } + + // Iterate over the remaining slices, skipping the one used as countMap + for i, result := range results { + if i == minIdx { + continue + } + + currentCount := make(map[string]bool) + for _, str := range result { + if _, exists := countMap[str]; exists { + currentCount[str] = true + } + } + + // Remove elements from countMap that are not in currentCount + for key := range countMap { + if _, exists := currentCount[key]; !exists { + delete(countMap, key) + } + } + } + + resp := make([]string, 0, len(countMap)) + for str := range countMap { + resp = append(resp, str) + } + + return resp +} + +func composeSDiff(responses ...ops.StoreResponse) interface{} { + sort.Slice(responses, func(i, j int) bool { + return responses[i].SeqID < responses[j].SeqID + }) + + results := [][]string{} + minIdx := 0 + for idx := range responses { + if responses[idx].EvalResponse.Error != nil { + return responses[idx].EvalResponse.Error + } + results = append(results, responses[idx].EvalResponse.Result.([]string)) + } + + // Create the initial countMap from the smallest slice + countMap := make(map[string]int) + for _, str := range results[0] { + countMap[str] = 1 + } + + // Iterate over the remaining slices, skipping the one used as countMap + for i, result := range results { + if i == minIdx { + continue + } + for _, str := range result { + if _, exists := countMap[str]; exists { + countMap[str]++ + } + } + } + + resp := make([]string, 0, len(countMap)) + for str := range countMap { + if countMap[str] == 1 { + resp = append(resp, str) + } + } + + return resp +} + +func composeJSONMget(responses ...ops.StoreResponse) interface{} { + sort.Slice(responses, func(i, j int) bool { + return responses[i].SeqID < responses[j].SeqID + }) + + results := []interface{}{} + for idx := range responses { + if responses[idx].EvalResponse.Error != nil { + return responses[idx].EvalResponse.Error + } + results = append(results, responses[idx].EvalResponse.Result) + } + return results +} diff --git a/internal/worker/cmd_decompose.go b/internal/worker/cmd_decompose.go index e5035dff2..9a5f65a53 100644 --- a/internal/worker/cmd_decompose.go +++ b/internal/worker/cmd_decompose.go @@ -4,8 +4,11 @@ import ( "context" "log/slog" + "github.com/dicedb/dice/internal/clientio" "github.com/dicedb/dice/internal/cmd" diceerrors "github.com/dicedb/dice/internal/errors" + "github.com/dicedb/dice/internal/object" + "github.com/dicedb/dice/internal/ops" "github.com/dicedb/dice/internal/store" ) @@ -63,32 +66,36 @@ func decomposeRename(ctx context.Context, w *BaseWorker, cd *cmd.DiceDBCmd) ([]* // sets the value to the destination key using a SET command. func decomposeCopy(ctx context.Context, w *BaseWorker, cd *cmd.DiceDBCmd) ([]*cmd.DiceDBCmd, error) { // Waiting for GET command response - var val string + var resp *ops.StoreResponse select { case <-ctx.Done(): slog.Error("Timed out waiting for response from shards", slog.String("workerID", w.id), slog.Any("error", ctx.Err())) case preProcessedResp, ok := <-w.preprocessingChan: if ok { - evalResp := preProcessedResp.EvalResponse - if evalResp.Error != nil { - return nil, evalResp.Error - } - - val = evalResp.Result.(string) + resp = preProcessedResp } } - if len(cd.Args) != 2 { + if resp.EvalResponse.Error != nil || resp.EvalResponse.Result == clientio.IntegerZero { + return nil, &diceerrors.PreProcessError{Result: clientio.IntegerZero} + } + + if len(cd.Args) < 2 { return nil, diceerrors.ErrWrongArgumentCount("COPY") } - decomposedCmds := []*cmd.DiceDBCmd{} - decomposedCmds = append(decomposedCmds, - &cmd.DiceDBCmd{ - Cmd: store.Set, - Args: []string{cd.Args[1], val}, + newObj, ok := resp.EvalResponse.Result.(*object.InternalObj) + if !ok { + return nil, diceerrors.ErrInternalServer + } + + decomposedCmds := []*cmd.DiceDBCmd{ + { + Cmd: "OBJECTCOPY", + Args: cd.Args[1:], + InternalObj: newObj, }, - ) + } return decomposedCmds, nil } @@ -135,3 +142,54 @@ func decomposeMGet(_ context.Context, _ *BaseWorker, cd *cmd.DiceDBCmd) ([]*cmd. } return decomposedCmds, nil } + +func decomposeSInter(_ context.Context, _ *BaseWorker, cd *cmd.DiceDBCmd) ([]*cmd.DiceDBCmd, error) { + if len(cd.Args) < 1 { + return nil, diceerrors.ErrWrongArgumentCount("SINTER") + } + decomposedCmds := make([]*cmd.DiceDBCmd, 0, len(cd.Args)) + for i := 0; i < len(cd.Args); i++ { + decomposedCmds = append(decomposedCmds, + &cmd.DiceDBCmd{ + Cmd: store.Smembers, + Args: []string{cd.Args[i]}, + }, + ) + } + return decomposedCmds, nil +} + +func decomposeSDiff(_ context.Context, _ *BaseWorker, cd *cmd.DiceDBCmd) ([]*cmd.DiceDBCmd, error) { + if len(cd.Args) < 1 { + return nil, diceerrors.ErrWrongArgumentCount("SDIFF") + } + decomposedCmds := make([]*cmd.DiceDBCmd, 0, len(cd.Args)) + for i := 0; i < len(cd.Args); i++ { + decomposedCmds = append(decomposedCmds, + &cmd.DiceDBCmd{ + Cmd: store.Smembers, + Args: []string{cd.Args[i]}, + }, + ) + } + return decomposedCmds, nil +} + +func decomposeJSONMget(_ context.Context, _ *BaseWorker, cd *cmd.DiceDBCmd) ([]*cmd.DiceDBCmd, error) { + if len(cd.Args) < 2 { + return nil, diceerrors.ErrWrongArgumentCount("JSON.MGET") + } + + pattern := cd.Args[len(cd.Args)-1] + + decomposedCmds := make([]*cmd.DiceDBCmd, 0, len(cd.Args)) + for i := 0; i < len(cd.Args)-1; i++ { + decomposedCmds = append(decomposedCmds, + &cmd.DiceDBCmd{ + Cmd: store.JSONGet, + Args: []string{cd.Args[i], pattern}, + }, + ) + } + return decomposedCmds, nil +} diff --git a/internal/worker/cmd_meta.go b/internal/worker/cmd_meta.go index aac42d2b7..9119b15a1 100644 --- a/internal/worker/cmd_meta.go +++ b/internal/worker/cmd_meta.go @@ -55,15 +55,21 @@ const ( CmdGetSet = "GETSET" CmdGetEx = "GETEX" CmdGetDel = "GETDEL" + CmdLrange = "LRANGE" + CmdLinsert = "LINSERT" CmdJSONArrAppend = "JSON.ARRAPPEND" CmdJSONArrLen = "JSON.ARRLEN" CmdJSONArrPop = "JSON.ARRPOP" - CmdLrange = "LRANGE" - CmdLinsert = "LINSERT" - CmdJSONForget = "JSON.FORGET" + CmdJSONClear = "JSON.CLEAR" CmdJSONDel = "JSON.DEL" - CmdJSONToggle = "JSON.TOGGLE" + CmdJSONForget = "JSON.FORGET" + CmdJSONGet = "JSON.GET" + CmdJSONStrlen = "JSON.STRLEN" + CmdJSONObjlen = "JSON.OBJLEN" CmdJSONNumIncrBY = "JSON.NUMINCRBY" + CmdJSONNumMultBy = "JSON.NUMMULTBY" + CmdJSONType = "JSON.TYPE" + CmdJSONToggle = "JSON.TOGGLE" CmdJSONNumMultBY = "JSON.NUMMULTBY" CmdLPush = "LPUSH" CmdRPush = "RPUSH" @@ -74,8 +80,12 @@ const ( // Multi-shard commands. const ( - CmdMset = "MSET" - CmdMget = "MGET" + CmdMset = "MSET" + CmdMget = "MGET" + CmdSInter = "SINTER" + CmdSDiff = "SDIFF" + CmdJSONMget = "JSON.MGET" + CmdKeys = "KEYS" ) // Multi-Step-Multi-Shard commands @@ -86,21 +96,10 @@ const ( // Watch commands const ( - CmdSadd = "SADD" - CmdSrem = "SREM" - CmdScard = "SCARD" - CmdSmembers = "SMEMBERS" - - CmdGetWatch = "GET.WATCH" - CmdGetUnWatch = "GET.UNWATCH" - CmdZRangeWatch = "ZRANGE.WATCH" CmdHExists = "HEXISTS" CmdHKeys = "HKEYS" CmdHVals = "HVALS" CmdZPopMin = "ZPOPMIN" - CmdJSONClear = "JSON.CLEAR" - CmdJSONStrlen = "JSON.STRLEN" - CmdJSONObjlen = "JSON.OBJLEN" CmdZAdd = "ZADD" CmdZRange = "ZRANGE" CmdZRank = "ZRANK" @@ -149,6 +148,19 @@ const ( CmdBitField = "BITFIELD" CmdBitPos = "BITPOS" CmdBitFieldRO = "BITFIELD_RO" + CmdSadd = "SADD" + CmdSrem = "SREM" + CmdScard = "SCARD" + CmdSmembers = "SMEMBERS" + CmdDump = "DUMP" + CmdRestore = "RESTORE" +) + +// Watch commands +const ( + CmdGetWatch = "GET.WATCH" + CmdGetUnWatch = "GET.UNWATCH" + CmdZRangeWatch = "ZRANGE.WATCH" ) type CmdMeta struct { @@ -169,14 +181,14 @@ type CmdMeta struct { // If set to true, it signals that a preliminary step (such as fetching values from shards) // is necessary before the main command is executed. This is important for commands that depend // on the current state of data in the database. - preProcessingReq bool + preProcessing bool // preProcessResponse is a function that handles the preprocessing of a DiceDB command by // preparing the necessary operations (e.g., fetching values from shards) before the command // is executed. It takes the worker and the original DiceDB command as parameters and // ensures that any required information is retrieved and processed in advance. Use this when set // preProcessingReq = true. - preProcessResponse func(worker *BaseWorker, DiceDBCmd *cmd.DiceDBCmd) + preProcessResponse func(worker *BaseWorker, DiceDBCmd *cmd.DiceDBCmd) error } var CommandsMeta = map[string]CmdMeta{ @@ -241,10 +253,16 @@ var CommandsMeta = map[string]CmdMeta{ CmdJSONArrPop: { CmdType: SingleShard, }, - CmdGetRange: { + CmdJSONClear: { CmdType: SingleShard, }, - CmdJSONClear: { + CmdJSONDel: { + CmdType: SingleShard, + }, + CmdJSONForget: { + CmdType: SingleShard, + }, + CmdJSONGet: { CmdType: SingleShard, }, CmdJSONStrlen: { @@ -253,6 +271,21 @@ var CommandsMeta = map[string]CmdMeta{ CmdJSONObjlen: { CmdType: SingleShard, }, + CmdJSONNumIncrBY: { + CmdType: SingleShard, + }, + CmdJSONNumMultBy: { + CmdType: SingleShard, + }, + CmdJSONType: { + CmdType: SingleShard, + }, + CmdJSONToggle: { + CmdType: SingleShard, + }, + CmdGetRange: { + CmdType: SingleShard, + }, CmdPFAdd: { CmdType: SingleShard, }, @@ -310,21 +343,6 @@ var CommandsMeta = map[string]CmdMeta{ CmdLinsert: { CmdType: SingleShard, }, - CmdJSONForget: { - CmdType: SingleShard, - }, - CmdJSONDel: { - CmdType: SingleShard, - }, - CmdJSONToggle: { - CmdType: SingleShard, - }, - CmdJSONNumIncrBY: { - CmdType: SingleShard, - }, - CmdJSONNumMultBY: { - CmdType: SingleShard, - }, CmdLPush: { CmdType: SingleShard, }, @@ -340,35 +358,6 @@ var CommandsMeta = map[string]CmdMeta{ CmdLLEN: { CmdType: SingleShard, }, - - // Multi-shard commands. - CmdRename: { - CmdType: MultiShard, - preProcessingReq: true, - preProcessResponse: preProcessRename, - decomposeCommand: decomposeRename, - composeResponse: composeRename, - }, - - CmdCopy: { - CmdType: MultiShard, - preProcessingReq: true, - preProcessResponse: preProcessCopy, - decomposeCommand: decomposeCopy, - composeResponse: composeCopy, - }, - - CmdMset: { - CmdType: MultiShard, - decomposeCommand: decomposeMSet, - composeResponse: composeMSet, - }, - - CmdMget: { - CmdType: MultiShard, - decomposeCommand: decomposeMGet, - composeResponse: composeMGet, - }, CmdCMSQuery: { CmdType: SingleShard, }, @@ -405,31 +394,6 @@ var CommandsMeta = map[string]CmdMeta{ CmdHMGet: { CmdType: SingleShard, }, - - // Custom commands. - CmdAbort: { - CmdType: Custom, - }, - CmdAuth: { - CmdType: Custom, - }, - - // Watch commands - CmdGetWatch: { - CmdType: Watch, - }, - CmdZRangeWatch: { - CmdType: Watch, - }, - CmdPFCountWatch: { - CmdType: Watch, - }, - - // Unwatch commands - CmdGetUnWatch: { - CmdType: Unwatch, - }, - // Sorted set commands CmdZAdd: { CmdType: SingleShard, @@ -473,7 +437,6 @@ var CommandsMeta = map[string]CmdMeta{ CmdZPopMax: { CmdType: SingleShard, }, - // Bloom Filter CmdBFAdd: { CmdType: SingleShard, @@ -487,6 +450,83 @@ var CommandsMeta = map[string]CmdMeta{ CmdBFReserve: { CmdType: SingleShard, }, + CmdDump: { + CmdType: SingleShard, + }, + CmdRestore: { + CmdType: SingleShard, + }, + + // Multi-shard commands. + CmdRename: { + CmdType: MultiShard, + preProcessing: true, + preProcessResponse: preProcessRename, + decomposeCommand: decomposeRename, + composeResponse: composeRename, + }, + + CmdCopy: { + CmdType: MultiShard, + preProcessing: true, + preProcessResponse: customProcessCopy, + decomposeCommand: decomposeCopy, + composeResponse: composeCopy, + }, + + CmdMset: { + CmdType: MultiShard, + decomposeCommand: decomposeMSet, + composeResponse: composeMSet, + }, + + CmdMget: { + CmdType: MultiShard, + decomposeCommand: decomposeMGet, + composeResponse: composeMGet, + }, + + CmdSInter: { + CmdType: MultiShard, + decomposeCommand: decomposeSInter, + composeResponse: composeSInter, + }, + + CmdSDiff: { + CmdType: MultiShard, + decomposeCommand: decomposeSDiff, + composeResponse: composeSDiff, + }, + + CmdJSONMget: { + CmdType: MultiShard, + decomposeCommand: decomposeJSONMget, + composeResponse: composeJSONMget, + }, + + // Custom commands. + CmdAbort: { + CmdType: Custom, + }, + CmdAuth: { + CmdType: Custom, + }, + + // Watch commands + CmdGetWatch: { + CmdType: Watch, + }, + CmdZRangeWatch: { + CmdType: Watch, + }, + CmdPFCountWatch: { + CmdType: Watch, + }, + + // Unwatch commands + CmdGetUnWatch: { + CmdType: Unwatch, + }, } func init() { diff --git a/internal/worker/cmd_preprocess.go b/internal/worker/cmd_preprocess.go index 424f24ed4..0db633d24 100644 --- a/internal/worker/cmd_preprocess.go +++ b/internal/worker/cmd_preprocess.go @@ -2,18 +2,23 @@ package worker import ( "github.com/dicedb/dice/internal/cmd" + diceerrors "github.com/dicedb/dice/internal/errors" "github.com/dicedb/dice/internal/ops" ) // preProcessRename prepares the RENAME command for preprocessing by sending a GET command // to retrieve the value of the original key. The retrieved value is used later in the // decomposeRename function to delete the old key and set the new key. -func preProcessRename(w *BaseWorker, diceDBCmd *cmd.DiceDBCmd) { +func preProcessRename(w *BaseWorker, diceDBCmd *cmd.DiceDBCmd) error { + if len(diceDBCmd.Args) < 2 { + return diceerrors.ErrWrongArgumentCount("RENAME") + } + key := diceDBCmd.Args[0] sid, rc := w.shardManager.GetShardInfo(key) preCmd := cmd.DiceDBCmd{ - Cmd: CmdGet, + Cmd: "RENAME", Args: []string{key}, } @@ -26,27 +31,35 @@ func preProcessRename(w *BaseWorker, diceDBCmd *cmd.DiceDBCmd) { Client: nil, PreProcessing: true, } + + return nil } // preProcessCopy prepares the COPY command for preprocessing by sending a GET command // to retrieve the value of the original key. The retrieved value is used later in the // decomposeCopy function to copy the value to the destination key. -func preProcessCopy(w *BaseWorker, diceDBCmd *cmd.DiceDBCmd) { - key := diceDBCmd.Args[0] - sid, rc := w.shardManager.GetShardInfo(key) +func customProcessCopy(w *BaseWorker, diceDBCmd *cmd.DiceDBCmd) error { + if len(diceDBCmd.Args) < 2 { + return diceerrors.ErrWrongArgumentCount("COPY") + } - preCmd := cmd.DiceDBCmd{ - Cmd: CmdGet, - Args: []string{key}, + sid, rc := w.shardManager.GetShardInfo(diceDBCmd.Args[0]) + + preCmdk := cmd.DiceDBCmd{ + Cmd: "COPY", + Args: []string{diceDBCmd.Args[0]}, } + // Need to get response from both keys to handle Replace or not rc <- &ops.StoreOp{ SeqID: 0, RequestID: GenerateUniqueRequestID(), - Cmd: &preCmd, + Cmd: &preCmdk, WorkerID: w.id, ShardID: sid, Client: nil, PreProcessing: true, } + + return nil } diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 8a33c28e5..49b265f00 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -170,8 +170,13 @@ func (w *BaseWorker) Start(ctx context.Context) error { func (w *BaseWorker) executeCommandHandler(execCtx context.Context, errChan chan error, cmds []*cmd.DiceDBCmd, isWatchNotification bool) { // Retrieve metadata for the command to determine if multisharding is supported. meta, ok := CommandsMeta[cmds[0].Cmd] - if ok && meta.preProcessingReq { - meta.preProcessResponse(w, cmds[0]) + if ok && meta.preProcessing { + if err := meta.preProcessResponse(w, cmds[0]); err != nil { + e := w.ioHandler.Write(execCtx, err) + if e != nil { + slog.Debug("Error executing for worker", slog.String("workerID", w.id), slog.Any("error", err)) + } + } } err := w.executeCommand(execCtx, cmds[0], isWatchNotification) @@ -212,7 +217,13 @@ func (w *BaseWorker) executeCommand(ctx context.Context, diceDBCmd *cmd.DiceDBCm // If the command supports multisharding, break it down into multiple commands. cmdList, err = meta.decomposeCommand(ctx, w, diceDBCmd) if err != nil { - workerErr := w.ioHandler.Write(ctx, err) + var workerErr error + // Check if it's a CustomError + if customErr, ok := err.(*diceerrors.PreProcessError); ok { + workerErr = w.ioHandler.Write(ctx, customErr.Result) + } else { + workerErr = w.ioHandler.Write(ctx, err) + } if workerErr != nil { slog.Debug("Error executing for worker", slog.String("workerID", w.id), slog.Any("error", workerErr)) }