fix(multiple): october 2024 wave (#558)

* fix(multiple): october 2024 wave

* feat(middleware): serve stale content if not strict

* fix(ci): mod

* feat(deps): bump storages version

* fix(chore): stale-if-error

* feat(global): implement simplefs storage & add missing cdn props support

* fix(deny-request): text/event-stream drop closes #562

* feat(caddy): allow to configure TLSConfig in Caddyfile

* fix(api): delete by regexp

* bump(deps): use storages core v0.0.11

* fix(traefik): resolves #558, full rewrite to match the actual cache behavior. Yaegi sucks!

* fix(chore): bypass on some cases

configurationtypes/types.go

@@ -246,6 +246,7 @@ type DefaultCache struct {
 	Redis               CacheProvider `json:"redis" yaml:"redis"`
 	Port                Port          `json:"port" yaml:"port"`
 	Regex               Regex         `json:"regex" yaml:"regex"`
+	SimpleFS            CacheProvider `json:"simplefs" yaml:"simplefs"`
 	Stale               Duration      `json:"stale" yaml:"stale"`
 	Storers             []string      `json:"storers" yaml:"storers"`
 	Timeout             Timeout       `json:"timeout" yaml:"timeout"`
@@ -300,7 +301,7 @@ func (d *DefaultCache) GetMode() string {
 	return d.Mode
 }
 
-// GetNats returns nuts configuration
+// GetNats returns nats configuration
 func (d *DefaultCache) GetNats() CacheProvider {
 	return d.Nats
 }
@@ -340,6 +341,11 @@ func (d *DefaultCache) GetTTL() time.Duration {
 	return d.TTL.Duration
 }
 
+// GetSimpleFS returns simplefs configuration
+func (d *DefaultCache) GetSimpleFS() CacheProvider {
+	return d.SimpleFS
+}
+
 // GetStale returns the stale duration
 func (d *DefaultCache) GetStale() time.Duration {
 	return d.Stale.Duration
@@ -382,6 +388,7 @@ type DefaultCacheInterface interface {
 	GetHeaders() []string
 	GetKey() Key
 	GetRegex() Regex
+	GetSimpleFS() CacheProvider
 	GetStale() time.Duration
 	GetStorers() []string
 	GetTimeout() Timeout

go.mod

@@ -5,7 +5,7 @@ go 1.22.1
 require (
 	github.com/caddyserver/caddy/v2 v2.8.4
 	github.com/cespare/xxhash/v2 v2.2.0
-	github.com/darkweak/storages/core v0.0.8
+	github.com/darkweak/storages/core v0.0.11
 	github.com/google/uuid v1.6.0
 	github.com/pierrec/lz4/v4 v4.1.21
 	github.com/pquerna/cachecontrol v0.2.0

go.sum

@@ -98,8 +98,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns=
 github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654=
-github.com/darkweak/storages/core v0.0.8 h1:9e7rOxHiJwnvADDVCZ7LFRnUnOHGT+UMpNOFlR8BOiw=
-github.com/darkweak/storages/core v0.0.8/go.mod h1:ajTpB9IFLRIRY0EEFLjM5vtsrcNTh+TJK9yRxgG5/wY=
+github.com/darkweak/storages/core v0.0.11 h1:IwvpAtkhOmxC5pIffJ8opW6erpTnIi5zqPveiAQs8ew=
+github.com/darkweak/storages/core v0.0.11/go.mod h1:ajTpB9IFLRIRY0EEFLjM5vtsrcNTh+TJK9yRxgG5/wY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/api/souin.go

@@ -158,6 +158,7 @@ var storageToInfiniteTTLMap = map[string]time.Duration{
 	"OLRIC":                  types.OneYearDuration,
 	"OTTER":                  types.OneYearDuration,
 	"REDIS":                  -1,
+	"SIMPLEFS":               0,
 	types.DefaultStorageName: types.OneYearDuration,
 }
 
@@ -305,7 +306,9 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) {
 				s.purgeMapping()
 			} else {
 				submatch := keysRg.FindAllStringSubmatch(r.RequestURI, -1)[0][1]
-				s.BulkDelete(submatch, true)
+				for _, current := range s.storers {
+					current.DeleteMany(submatch)
+				}
 			}
 		} else {
 			ck, _ := s.surrogateStorage.Purge(r.Header)

pkg/middleware/middleware.go

@@ -67,7 +67,7 @@ func NewHTTPCacheHandler(c configurationtypes.AbstractConfigurationInterface) *S
 	storers := []types.Storer{}
 	if len(storedStorers) != 0 {
 		dc := c.GetDefaultCache()
-		for _, s := range []string{dc.GetBadger().Uuid, dc.GetEtcd().Uuid, dc.GetNats().Uuid, dc.GetNuts().Uuid, dc.GetOlric().Uuid, dc.GetOtter().Uuid, dc.GetRedis().Uuid} {
+		for _, s := range []string{dc.GetBadger().Uuid, dc.GetEtcd().Uuid, dc.GetNats().Uuid, dc.GetNuts().Uuid, dc.GetOlric().Uuid, dc.GetOtter().Uuid, dc.GetRedis().Uuid, dc.GetSimpleFS().Uuid} {
 			if s != "" {
 				if st := core.GetRegisteredStorer(s); st != nil {
 					storers = append(storers, st.(types.Storer))
@@ -244,7 +244,7 @@ func (s *SouinBaseHandler) Store(
 		ma = time.Duration(responseCc.SMaxAge) * time.Second
 	} else if responseCc.MaxAge >= 0 {
 		ma = time.Duration(responseCc.MaxAge) * time.Second
-	} else if customWriter.Header().Get("Expires") != "" {
+	} else if !modeContext.Bypass_response && customWriter.Header().Get("Expires") != "" {
 		exp, err := time.Parse(time.RFC1123, customWriter.Header().Get("Expires"))
 		if err != nil {
 			return nil
@@ -569,7 +569,7 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 
 	req := s.context.SetBaseContext(rq)
 	cacheName := req.Context().Value(context.CacheName).(string)
-	if rq.Header.Get("Upgrade") == "websocket" || (s.ExcludeRegex != nil && s.ExcludeRegex.MatchString(rq.RequestURI)) {
+	if rq.Header.Get("Upgrade") == "websocket" || rq.Header.Get("Accept") == "text/event-stream" || (s.ExcludeRegex != nil && s.ExcludeRegex.MatchString(rq.RequestURI)) {
 		rw.Header().Set("Cache-Status", cacheName+"; fwd=bypass; detail=EXCLUDED-REQUEST-URI")
 		return next(rw, req)
 	}
@@ -673,13 +673,13 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 				return nil
 			}
 
-			if validator.NeedRevalidation {
+			if !modeContext.Bypass_request && validator.NeedRevalidation {
 				err := s.Revalidate(validator, next, customWriter, req, requestCc, cachedKey, uri)
 				_, _ = customWriter.Send()
 
 				return err
 			}
-			if resCc, _ := cacheobject.ParseResponseCacheControl(rfc.HeaderAllCommaSepValuesString(response.Header, headerName)); resCc.NoCachePresent {
+			if resCc, _ := cacheobject.ParseResponseCacheControl(rfc.HeaderAllCommaSepValuesString(response.Header, headerName)); !modeContext.Bypass_response && resCc.NoCachePresent {
 				prometheus.Increment(prometheus.NoCachedResponseCounter)
 				err := s.Revalidate(validator, next, customWriter, req, requestCc, cachedKey, uri)
 				_, _ = customWriter.Send()
@@ -726,7 +726,7 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 					return err
 				}
 
-				if responseCc.MustRevalidate || responseCc.NoCachePresent || validator.NeedRevalidation {
+				if modeContext.Bypass_response || responseCc.MustRevalidate || responseCc.NoCachePresent || validator.NeedRevalidation {
 					req.Header["If-None-Match"] = append(req.Header["If-None-Match"], validator.ResponseETag)
 					err := s.Revalidate(validator, next, customWriter, req, requestCc, cachedKey, uri)
 					statusCode := customWriter.GetStatusCode()
@@ -737,6 +737,7 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 							response.Header.Set("Cache-Status", response.Header.Get("Cache-Status")+code)
 							maps.Copy(customWriter.Header(), response.Header)
 							customWriter.WriteHeader(response.StatusCode)
+							customWriter.Buf.Reset()
 							_, _ = io.Copy(customWriter.Buf, response.Body)
 							_, err := customWriter.Send()
 
@@ -774,7 +775,7 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 					return err
 				}
 
-				if rfc.ValidateMaxAgeCachedStaleResponse(requestCc, response, int(addTime.Seconds())) != nil {
+				if !modeContext.Strict || rfc.ValidateMaxAgeCachedStaleResponse(requestCc, responseCc, response, int(addTime.Seconds())) != nil {
 					customWriter.WriteHeader(response.StatusCode)
 					rfc.HitStaleCache(&response.Header)
 					maps.Copy(customWriter.Header(), response.Header)
@@ -784,6 +785,47 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 					return err
 				}
 			}
+		} else if stale != nil {
+			response := stale
+
+			if !modeContext.Strict {
+				rfc.SetCacheStatusHeader(response, storerName)
+				customWriter.WriteHeader(response.StatusCode)
+				rfc.HitStaleCache(&response.Header)
+				maps.Copy(customWriter.Header(), response.Header)
+				_, _ = io.Copy(customWriter.Buf, response.Body)
+				_, err := customWriter.Send()
+
+				return err
+			}
+
+			addTime, _ := time.ParseDuration(response.Header.Get(rfc.StoredTTLHeader))
+			responseCc, _ := cacheobject.ParseResponseCacheControl(rfc.HeaderAllCommaSepValuesString(response.Header, "Cache-Control"))
+
+			if !modeContext.Strict || rfc.ValidateMaxAgeCachedStaleResponse(requestCc, responseCc, response, int(addTime.Seconds())) != nil {
+				_, _ = time.ParseDuration(response.Header.Get(rfc.StoredTTLHeader))
+				rfc.SetCacheStatusHeader(response, storerName)
+
+				responseCc, _ := cacheobject.ParseResponseCacheControl(rfc.HeaderAllCommaSepValuesString(response.Header, "Cache-Control"))
+
+				if responseCc.StaleIfError > -1 || requestCc.StaleIfError > 0 {
+					err := s.Revalidate(validator, next, customWriter, req, requestCc, cachedKey, uri)
+					statusCode := customWriter.GetStatusCode()
+					if err != nil {
+						code := fmt.Sprintf("; fwd-status=%d", statusCode)
+						rfc.HitStaleCache(&response.Header)
+						response.Header.Set("Cache-Status", response.Header.Get("Cache-Status")+code)
+						maps.Copy(customWriter.Header(), response.Header)
+						customWriter.WriteHeader(response.StatusCode)
+						customWriter.Buf.Reset()
+						_, _ = io.Copy(customWriter.Buf, response.Body)
+						_, err := customWriter.Send()
+
+						return err
+					}
+				}
+
+			}
 		}
 	}
 

pkg/rfc/age.go

@@ -31,11 +31,25 @@ func ValidateMaxAgeCachedResponse(co *cacheobject.RequestCacheDirectives, res *h
 	return validateMaxAgeCachedResponse(res, int(ma), 0)
 }
 
-func ValidateMaxAgeCachedStaleResponse(co *cacheobject.RequestCacheDirectives, res *http.Response, addTime int) *http.Response {
+func ValidateMaxAgeCachedStaleResponse(co *cacheobject.RequestCacheDirectives, resCo *cacheobject.ResponseCacheDirectives, res *http.Response, addTime int) *http.Response {
 	if co.MaxStaleSet {
 		return res
 	}
 
+	if resCo != nil && (resCo.StaleIfError > -1 || co.StaleIfError > 0) {
+		if resCo.StaleIfError > -1 {
+			if response := validateMaxAgeCachedResponse(res, int(resCo.StaleIfError), addTime); response != nil {
+				return response
+			}
+		}
+
+		if co.StaleIfError > 0 {
+			if response := validateMaxAgeCachedResponse(res, int(co.StaleIfError), addTime); response != nil {
+				return response
+			}
+		}
+	}
+
 	if co.MaxStale < 0 {
 		return nil
 	}

pkg/rfc/age_test.go

@@ -63,20 +63,20 @@ func Test_ValidateMaxStaleCachedResponse(t *testing.T) {
 		},
 	}
 
-	if ValidateMaxAgeCachedStaleResponse(&coWithoutMaxStale, &expiredMaxAge, 3) != nil {
+	if ValidateMaxAgeCachedStaleResponse(&coWithoutMaxStale, nil, &expiredMaxAge, 3) != nil {
 		t.Errorf("The max-stale validation should return nil instead of the response with the given parameters:\nRequestCacheDirectives: %+v\nResponse: %+v\n", coWithoutMaxStale, expiredMaxAge)
 	}
-	if ValidateMaxAgeCachedStaleResponse(&coWithoutMaxStale, &validMaxAge, 14) != nil {
+	if ValidateMaxAgeCachedStaleResponse(&coWithoutMaxStale, nil, &validMaxAge, 14) != nil {
 		t.Errorf("The max-stale validation should return the response instead of nil with the given parameters:\nRequestCacheDirectives: %+v\nResponse: %+v\n", coWithoutMaxStale, validMaxAge)
 	}
 
-	if ValidateMaxAgeCachedStaleResponse(&coWithMaxStale, &expiredMaxAge, 0) != nil {
+	if ValidateMaxAgeCachedStaleResponse(&coWithMaxStale, nil, &expiredMaxAge, 0) != nil {
 		t.Errorf("The max-stale validation should return nil instead of the response with the given parameters:\nRequestCacheDirectives: %+v\nResponse: %+v\n", coWithMaxStale, expiredMaxAge)
 	}
-	if ValidateMaxAgeCachedStaleResponse(&coWithMaxStaleSet, &expiredMaxAge, 0) == nil {
+	if ValidateMaxAgeCachedStaleResponse(&coWithMaxStaleSet, nil, &expiredMaxAge, 0) == nil {
 		t.Errorf("The max-stale validation should return the response instead of nil with the given parameters:\nRequestCacheDirectives: %+v\nResponse: %+v\n", coWithMaxStaleSet, expiredMaxAge)
 	}
-	if ValidateMaxAgeCachedStaleResponse(&coWithMaxStale, &validMaxAge, 5) == nil {
+	if ValidateMaxAgeCachedStaleResponse(&coWithMaxStale, nil, &validMaxAge, 5) == nil {
 		t.Errorf("The max-stale validation should return the response instead of nil with the given parameters:\nRequestCacheDirectives: %+v\nResponse: %+v\n", coWithMaxStale, expiredMaxAge)
 	}
 }

pkg/surrogate/providers/common.go

@@ -37,11 +37,17 @@ var storageToInfiniteTTLMap = map[string]time.Duration{
 	"OLRIC":                  types.OneYearDuration,
 	"OTTER":                  types.OneYearDuration,
 	"REDIS":                  -1,
+	"SIMPLEFS":               0,
 	types.DefaultStorageName: types.OneYearDuration,
 }
 
 func (s *baseStorage) ParseHeaders(value string) []string {
-	return strings.Split(value, s.parent.getHeaderSeparator())
+	res := strings.Split(value, s.parent.getHeaderSeparator())
+	for i, v := range res {
+		res[i] = strings.TrimSpace(v)
+	}
+
+	return res
 }
 
 func getCandidateHeader(header http.Header, getCandidates func() []string) (string, string) {

plugins/beego/go.mod

@@ -7,8 +7,8 @@ toolchain go1.22.4
 require (
 	github.com/beego/beego/v2 v2.1.1
 	github.com/darkweak/souin v1.7.2
-	github.com/darkweak/souin/plugins/souin v0.0.0-00010101000000-000000000000
-	github.com/darkweak/souin/plugins/souin/storages v0.0.0-00010101000000-000000000000
+	github.com/darkweak/souin/plugins/souin v1.7.2
+	github.com/darkweak/souin/plugins/souin/storages v1.7.2
 )
 
 require (
@@ -41,7 +41,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/darkweak/go-esi v0.0.5 // indirect
 	github.com/darkweak/storages/badger v0.0.8 // indirect
-	github.com/darkweak/storages/core v0.0.8 // indirect
+	github.com/darkweak/storages/core v0.0.11 // indirect
 	github.com/darkweak/storages/etcd v0.0.8 // indirect
 	github.com/darkweak/storages/nats v0.0.8 // indirect
 	github.com/darkweak/storages/nuts v0.0.8 // indirect
@@ -186,5 +186,4 @@ require (
 replace (
 	github.com/darkweak/souin v1.7.2 => ../..
 	github.com/darkweak/souin/plugins/souin => ../souin
-	github.com/darkweak/souin/plugins/souin/storages => ../souin/storages
 )

plugins/beego/go.sum

@@ -141,10 +141,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns=
 github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654=
+github.com/darkweak/souin/plugins/souin/storages v1.7.2 h1:vA1oFap6sbWO+Ebbq6NGtjmCFuCRJOZeG+XXPhhSIWA=
+github.com/darkweak/souin/plugins/souin/storages v1.7.2/go.mod h1:VfkwGN+ubAuluSwbjGHqImbUjxdEA0N9xGJUTCcFBV0=
 github.com/darkweak/storages/badger v0.0.8 h1:rKVXrasVA74xgiqGRgW0kH11NUIlWwn9HiFyHUok85k=
 github.com/darkweak/storages/badger v0.0.8/go.mod h1:ZmrNmKkFzyu/B3+1nsvVeTvyg2I2mOV5yTpT46mZ06o=
-github.com/darkweak/storages/core v0.0.8 h1:9e7rOxHiJwnvADDVCZ7LFRnUnOHGT+UMpNOFlR8BOiw=
-github.com/darkweak/storages/core v0.0.8/go.mod h1:ajTpB9IFLRIRY0EEFLjM5vtsrcNTh+TJK9yRxgG5/wY=
+github.com/darkweak/storages/core v0.0.11 h1:IwvpAtkhOmxC5pIffJ8opW6erpTnIi5zqPveiAQs8ew=
+github.com/darkweak/storages/core v0.0.11/go.mod h1:ajTpB9IFLRIRY0EEFLjM5vtsrcNTh+TJK9yRxgG5/wY=
 github.com/darkweak/storages/etcd v0.0.8 h1:Guzv6zgxkQJLjak36KsbtQqkmwMRJoZZI0B7ztZKIik=
 github.com/darkweak/storages/etcd v0.0.8/go.mod h1:Yw9xJramKAzIRoC7tizVMYPSwUBHqxY5BPTh8OgyISY=
 github.com/darkweak/storages/nats v0.0.8 h1:HRS3i2zzzIq1Qb3yoOUWD6MoRQgGV1NbECF1ex4wZjg=