Skip to content

Commit 3410561

Browse files
author
Vincent Jordan
committed
feat(storers): Add a new NutsMemcached storage backend built on NutsDB
1 parent 3acd7f2 commit 3410561

File tree

4 files changed

+308
-0
lines changed

4 files changed

+308
-0
lines changed

‎configurationtypes/types.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ type DefaultCache struct {
224224
Etcd CacheProvider `json:"etcd" yaml:"etcd"`
225225
Mode string `json:"mode" yaml:"mode"`
226226
Nuts CacheProvider `json:"nuts" yaml:"nuts"`
227+
NutsMemcached CacheProvider `json:"nuts_memcached" yaml:"nuts_memcached"`
227228
Olric CacheProvider `json:"olric" yaml:"olric"`
228229
Redis CacheProvider `json:"redis" yaml:"redis"`
229230
Port Port `json:"port" yaml:"port"`
@@ -285,6 +286,11 @@ func (d *DefaultCache) GetNuts() CacheProvider {
285286
return d.Nuts
286287
}
287288

289+
// GetNutsMemcached returns nuts_memcached configuration
290+
func (d *DefaultCache) GetNutsMemcached() CacheProvider {
291+
return d.NutsMemcached
292+
}
293+
288294
// GetOlric returns olric configuration
289295
func (d *DefaultCache) GetOlric() CacheProvider {
290296
return d.Olric
@@ -335,6 +341,7 @@ type DefaultCacheInterface interface {
335341
GetEtcd() CacheProvider
336342
GetMode() string
337343
GetNuts() CacheProvider
344+
GetNutsMemcached() CacheProvider
338345
GetOlric() CacheProvider
339346
GetRedis() CacheProvider
340347
GetHeaders() []string

‎pkg/storage/nutsMemcachedProvider.go

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
package storage
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"encoding/json"
7+
"net/http"
8+
"strings"
9+
"time"
10+
11+
t "github.com/darkweak/souin/configurationtypes"
12+
"github.com/darkweak/souin/pkg/rfc"
13+
"github.com/darkweak/souin/pkg/storage/types"
14+
"github.com/imdario/mergo"
15+
"github.com/nutsdb/nutsdb"
16+
"go.uber.org/zap"
17+
)
18+
19+
var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{}
20+
21+
// NutsMemcached provider type
22+
type NutsMemcached struct {
23+
*nutsdb.DB
24+
stale time.Duration
25+
logger *zap.Logger
26+
}
27+
28+
// const (
29+
// bucket = "souin-bucket"
30+
// nutsLimit = 1 << 16
31+
// )
32+
33+
// func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
34+
// iotas := []string{"RWMode", "StartFileLoadingMode"}
35+
// for _, i := range iotas {
36+
// if v := m[i]; v != nil {
37+
// currentMode := nutsdb.FileIO
38+
// switch v {
39+
// case 1:
40+
// currentMode = nutsdb.MMap
41+
// }
42+
// m[i] = currentMode
43+
// }
44+
// }
45+
46+
// for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} {
47+
// if v := m[i]; v != nil {
48+
// m[i], _ = v.(int64)
49+
// }
50+
// }
51+
52+
// if v := m["EntryIdxMode"]; v != nil {
53+
// m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode
54+
// switch v {
55+
// case 1:
56+
// m["EntryIdxMode"] = nutsdb.HintKeyAndRAMIdxMode
57+
// }
58+
// }
59+
60+
// if v := m["SyncEnable"]; v != nil {
61+
// m["SyncEnable"] = true
62+
// if b, ok := v.(bool); ok {
63+
// m["SyncEnable"] = b
64+
// } else if s, ok := v.(string); ok {
65+
// m["SyncEnable"], _ = strconv.ParseBool(s)
66+
// }
67+
// }
68+
69+
// return m
70+
// }
71+
72+
// NutsConnectionFactory function create new Nuts instance
73+
func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) {
74+
dc := c.GetDefaultCache()
75+
nutsConfiguration := dc.GetNuts()
76+
nutsOptions := nutsdb.DefaultOptions
77+
nutsOptions.Dir = "/tmp/souin-nuts"
78+
if nutsConfiguration.Configuration != nil {
79+
var parsedNuts nutsdb.Options
80+
nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{}))
81+
if b, e := json.Marshal(nutsConfiguration.Configuration); e == nil {
82+
if e = json.Unmarshal(b, &parsedNuts); e != nil {
83+
c.GetLogger().Sugar().Error("Impossible to parse the configuration for the Nuts provider", e)
84+
}
85+
}
86+
87+
if err := mergo.Merge(&nutsOptions, parsedNuts, mergo.WithOverride); err != nil {
88+
c.GetLogger().Sugar().Error("An error occurred during the nutsOptions merge from the default options with your configuration.")
89+
}
90+
} else {
91+
nutsOptions.RWMode = nutsdb.MMap
92+
if nutsConfiguration.Path != "" {
93+
nutsOptions.Dir = nutsConfiguration.Path
94+
}
95+
}
96+
97+
if instance, ok := nutsMemcachedInstanceMap[nutsOptions.Dir]; ok && instance != nil {
98+
return &Nuts{
99+
DB: instance,
100+
stale: dc.GetStale(),
101+
logger: c.GetLogger(),
102+
}, nil
103+
}
104+
105+
db, e := nutsdb.Open(nutsOptions)
106+
107+
if e != nil {
108+
c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e)
109+
return nil, e
110+
}
111+
112+
instance := &Nuts{
113+
DB: db,
114+
stale: dc.GetStale(),
115+
logger: c.GetLogger(),
116+
}
117+
nutsMemcachedInstanceMap[nutsOptions.Dir] = instance.DB
118+
119+
return instance, nil
120+
}
121+
122+
// Name returns the storer name
123+
func (provider *NutsMemcached) Name() string {
124+
return "NUTS_MEMCACHED"
125+
}
126+
127+
// ListKeys method returns the list of existing keys
128+
func (provider *NutsMemcached) ListKeys() []string {
129+
keys := []string{}
130+
131+
e := provider.DB.View(func(tx *nutsdb.Tx) error {
132+
e, _ := tx.GetAll(bucket)
133+
for _, k := range e {
134+
if !strings.Contains(string(k.Key), surrogatePrefix) {
135+
keys = append(keys, string(k.Key))
136+
}
137+
}
138+
return nil
139+
})
140+
141+
if e != nil {
142+
return []string{}
143+
}
144+
145+
return keys
146+
}
147+
148+
// MapKeys method returns the map of existing keys
149+
func (provider *NutsMemcached) MapKeys(prefix string) map[string]string {
150+
keys := map[string]string{}
151+
152+
e := provider.DB.View(func(tx *nutsdb.Tx) error {
153+
e, _ := tx.GetAll(bucket)
154+
for _, k := range e {
155+
if strings.HasPrefix(string(k.Key), prefix) {
156+
nk, _ := strings.CutPrefix(string(k.Key), prefix)
157+
keys[nk] = string(k.Value)
158+
}
159+
}
160+
return nil
161+
})
162+
163+
if e != nil {
164+
return map[string]string{}
165+
}
166+
167+
return keys
168+
}
169+
170+
// Get method returns the populated response if exists, empty response then
171+
func (provider *NutsMemcached) Get(key string) (item []byte) {
172+
_ = provider.DB.View(func(tx *nutsdb.Tx) error {
173+
i, e := tx.Get(bucket, []byte(key))
174+
if i != nil {
175+
item = i.Value
176+
}
177+
return e
178+
})
179+
180+
return
181+
}
182+
183+
// Prefix method returns the populated response if exists, empty response then
184+
func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response {
185+
var result *http.Response
186+
187+
_ = provider.DB.View(func(tx *nutsdb.Tx) error {
188+
prefix := []byte(key)
189+
190+
if entries, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil {
191+
return err
192+
} else {
193+
for _, entry := range entries {
194+
if varyVoter(key, req, string(entry.Key)) {
195+
if res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(entry.Value)), req); err == nil {
196+
rfc.ValidateETag(res, validator)
197+
if validator.Matched {
198+
provider.logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(entry.Key), validator)
199+
result = res
200+
return nil
201+
}
202+
203+
provider.logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(entry.Key), validator)
204+
} else {
205+
provider.logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(entry.Key), err)
206+
}
207+
}
208+
}
209+
}
210+
return nil
211+
})
212+
213+
return result
214+
}
215+
216+
// Set method will store the response in Nuts provider
217+
func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration time.Duration) error {
218+
if duration == 0 {
219+
duration = url.TTL.Duration
220+
}
221+
222+
err := provider.DB.Update(func(tx *nutsdb.Tx) error {
223+
return tx.Put(bucket, []byte(key), value, uint32(duration.Seconds()))
224+
})
225+
226+
if err != nil {
227+
provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err)
228+
return err
229+
}
230+
231+
err = provider.DB.Update(func(tx *nutsdb.Tx) error {
232+
return tx.Put(bucket, []byte(StalePrefix+key), value, uint32((provider.stale + duration).Seconds()))
233+
})
234+
235+
if err != nil {
236+
provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err)
237+
}
238+
239+
return nil
240+
}
241+
242+
// Delete method will delete the response in Nuts provider if exists corresponding to key param
243+
func (provider *NutsMemcached) Delete(key string) {
244+
_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
245+
return tx.Delete(bucket, []byte(key))
246+
})
247+
}
248+
249+
// DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param
250+
func (provider *NutsMemcached) DeleteMany(key string) {
251+
_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
252+
if entries, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil {
253+
return err
254+
} else {
255+
for _, entry := range entries {
256+
_ = tx.Delete(bucket, entry.Key)
257+
}
258+
}
259+
return nil
260+
})
261+
}
262+
263+
// Init method will
264+
func (provider *NutsMemcached) Init() error {
265+
return nil
266+
}
267+
268+
// Reset method will reset or close provider
269+
func (provider *NutsMemcached) Reset() error {
270+
return provider.DB.Update(func(tx *nutsdb.Tx) error {
271+
return tx.DeleteBucket(1, bucket)
272+
})
273+
}

‎pkg/storage/storage.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ var storageMap = map[string]StorerInstanciator{
2626
"olric": OlricConnectionFactory,
2727
"embedded_olric": EmbeddedOlricConnectionFactory,
2828
"nuts": NutsConnectionFactory,
29+
"nuts_memcached": NutsMemcachedConnectionFactory,
2930
"badger": BadgerConnectionFactory,
3031
}
3132

@@ -44,6 +45,8 @@ func getStorageNameFromConfiguration(configuration configurationtypes.AbstractCo
4445
}
4546
} else if configuration.GetDefaultCache().GetNuts().Configuration != nil || configuration.GetDefaultCache().GetNuts().Path != "" {
4647
return "nuts"
48+
} else if configuration.GetDefaultCache().GetNutsMemcached().Configuration != nil || configuration.GetDefaultCache().GetNutsMemcached().Path != "" {
49+
return "nuts_memcached"
4750
}
4851

4952
return "badger"

‎plugins/caddy/configuration.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ type DefaultCache struct {
3838
Etcd configurationtypes.CacheProvider `json:"etcd"`
3939
// NutsDB provider configuration.
4040
Nuts configurationtypes.CacheProvider `json:"nuts"`
41+
// NutsMemcached provider configuration.
42+
NutsMemcached configurationtypes.CacheProvider `json:"nuts_memcached"`
4143
// Regex to exclude cache.
4244
Regex configurationtypes.Regex `json:"regex"`
4345
// Storage providers chaining and order.
@@ -100,6 +102,11 @@ func (d *DefaultCache) GetNuts() configurationtypes.CacheProvider {
100102
return d.Nuts
101103
}
102104

105+
// GetNutsMemcached returns nuts_memcached configuration
106+
func (d *DefaultCache) GetNutsMemcached() configurationtypes.CacheProvider {
107+
return d.NutsMemcached
108+
}
109+
103110
// GetOlric returns olric configuration
104111
func (d *DefaultCache) GetOlric() configurationtypes.CacheProvider {
105112
return d.Olric
@@ -476,6 +483,24 @@ func parseConfiguration(cfg *Configuration, h *caddyfile.Dispenser, isGlobal boo
476483
}
477484
}
478485
cfg.DefaultCache.Nuts = provider
486+
case "nuts_memcached":
487+
provider := configurationtypes.CacheProvider{}
488+
for nesting := h.Nesting(); h.NextBlock(nesting); {
489+
directive := h.Val()
490+
switch directive {
491+
case "url":
492+
urlArgs := h.RemainingArgs()
493+
provider.URL = urlArgs[0]
494+
case "path":
495+
urlArgs := h.RemainingArgs()
496+
provider.Path = urlArgs[0]
497+
case "configuration":
498+
provider.Configuration = parseCaddyfileRecursively(h)
499+
default:
500+
return h.Errf("unsupported nuts directive: %s", directive)
501+
}
502+
}
503+
cfg.DefaultCache.Nuts = provider
479504
case "olric":
480505
cfg.DefaultCache.Distributed = true
481506
provider := configurationtypes.CacheProvider{}

0 commit comments

Comments
 (0)