glue, manage api 2.0

Author: Krukov

README.md

@@ -52,15 +52,15 @@ Specify target host and timeout for forwarding requests
  3) prefix or proxying - `/my/*`
  4) mixed `/my/{parameter}/path/*`
 
- And need to define rules for our route: each rule heve `type` and options related to this kind of rule, 
+ And need to define rules for our route: each rule heve `kind` and options related to this kind of rule, 
 
  ```json
  {
   // ...
   "rules": {
     "/posts/{id}": [  // route
       {
-        "type": "cache",  // rule type "cache" - simple cache strategy
+        "kind": "cache",  // rule kind "cache" - simple cache strategy
         "ttl": "10s",  // cache ttl
         "key": "post-{id}",  // cache key template, can contain url parameter from route
         "storage": "main_ha"  // storage name from "storages" section that will be used for this rule
@@ -73,12 +73,12 @@ Specify target host and timeout for forwarding requests
  ```
  That is, you can run `circa` and it will cache requests to GET /posts/{id} for 10 seconds and proxy all other requests 
 
- There are a list of rules types and parameters for each:
+ There are a list of rules kinds and parameters for each:
 
 1. Cache with ttl
 ```json
 {
-    "type": "cache",  // by default
+    "kind": "cache",  // by default
     "ttl": "10m50s", // cache time to live  Valid time units are “ns”, “us” (or “µs”), “ms”, “s”, “m”, “h”.
     "key": "....."
 }
@@ -87,7 +87,7 @@ Specify target host and timeout for forwarding requests
 2. Cache with early expiration - 
 ```json
 {
-    "type": "early",
+    "kind": "early",
     "ttl": "10h", // cache time to live
     "early_ttl": "1h",  // time for pre invalidation
     "key": "....."
@@ -97,7 +97,7 @@ Specify target host and timeout for forwarding requests
 3. Cache with hit expiration - 
 ```json
 {
-    "type": "hit",
+    "kind": "hit",
     "ttl": "10h", // cache time to live
     "hits": 100,  // number of hits before cache will be invalidated
     "update_after": 10, // optional; number of hits for pre invalidation
@@ -108,7 +108,7 @@ Specify target host and timeout for forwarding requests
 4. Failover cache - 
 ```json
 {
-    "type": "fail",
+    "kind": "fail",
     "ttl": "10h", // cache time to live
     "key": "....."
 }
@@ -117,7 +117,7 @@ Specify target host and timeout for forwarding requests
 5. Rate limit - 
 ```json
 {
-    "type": "rate-limit",
+    "kind": "rate-limit",
     "ttl": "10m", // limit period
     "hits": 100, //  number of hits  ( read as 100 request per 10 min)
     "key": "....."
@@ -127,7 +127,7 @@ Specify target host and timeout for forwarding requests
 6. Retry - 
 ```json
 {
-    "type": "retry",
+    "kind": "retry",
     "methods": ["GET", "HEAD"],
     "backoff": "5s",
     "count": 5 // retry attempts
@@ -138,7 +138,7 @@ Specify target host and timeout for forwarding requests
 7. request_id -  `skip_return` - check that backend return response with tha same request ID
 ```json
 {
-    "type": "request_id",
+    "kind": "request_id",
     "methods": ["GET", "POST", "DELETE", "PUT", "PATCH"],
     "skip_return": false // by default
 }
@@ -147,7 +147,7 @@ Specify target host and timeout for forwarding requests
 8. idempotency
 ```json
 {
-    "type": "idempotency",
+    "kind": "idempotency",
     "ttl": "1m"
     // "key": "{R:body|hash}",
 }
@@ -156,7 +156,7 @@ Specify target host and timeout for forwarding requests
 9. invalidate
 ```json
 {
-    "type": "invalidate",
+    "kind": "invalidate",
     "methods": ["POST", "DELETE", "PUT", "PATCH"],
     "key": "posts-{id}:key"  // the key template that will be deleted after success request
 }
@@ -165,14 +165,14 @@ Specify target host and timeout for forwarding requests
 10. skip - will skip all rules see "Routing"
 ```json
 {
-    "type": "skip"
+    "kind": "skip"
 }
 ```
 
 11. proxy - rule to change method or target host for proxing 
 ```json
 {
-    "type": "proxy",
+    "kind": "proxy",
     "methods": ["GET"],  
     "path": "/posts/",  // optional
     "method": "POST",  // optional
@@ -184,20 +184,29 @@ Will proxy all get methods as post request to the https://google.com/posts/
 12. Glue
 ```json
 {
-  "type": "json",
-  "paths": ["/posts", "/articles"]
+  "kind": "glue",
+  "model": "single", // "list" 
+  "calls": {
+    "user_info": "/users/{id}/info",
+    "user_meta": "/users/{id}/meta",
+    "posts": "/posts/?user={id}"
+  },
+  "mapping": {
+    "posts": "posts"
+  }  // {..info, ..meta, "posts": posts}
 }
 
 TODO:
- - unix socket as target
- - proxy fast with no rules
- - glue
+  - proxy fast with no rules [x] - with /* rule - fast tpp
+ - cache resolve
+ - glue[partly]
+ - proxy header - like request-id just get and return back 
+ - cache condition - header, time, query 
  - proxy filebody
+ - proxy Location - bug
+ - manage api
  - circuit-breaker []
  - rate-limit with a sliding window
  - hot cache ?
- - hot reload with config change 
  - config flush
-
-ConfigRepo (store a config) -> Config (control a rules storages and sync configRepo) -> Runner 
-Request -> Resolver (route -> rules) -> Rule -> Handler  -> Requester -> Response
+ - unix socket as targe

config.json

@@ -6,22 +6,34 @@
   "rules": {
     "/posts/{id}": [
       {
-        "kind": "idempotency",
-        "key": "post-ide-{R:body|hash}",
-        "ttl": "10s",
-        "methods": ["POST"]
+        "kind": "glue",
+        "model": "object",
+        "calls": {
+          "post_main": "/posts/{id}",
+          "post_meta": "/post_meta/{id}"
+        }
       },
       {
-        "key": "posts-{id}:key",
-        "kind": "early",
-        "storage": "mem",
-        "early_ttl": "1m",
-        "ttl": "10m"
+        "kind": "invalidate",
+        "methods": ["DELETE", "PUT", "PATCH"],
+        "key": "posts-{id}:key"
       },
       {
         "kind": "invalidate",
-        "methods": ["POST", "DELETE", "PUT", "PATCH"],
-        "key": "posts-{id}:key"
+        "methods": ["DELETE", "PUT", "PATCH"],
+        "key": "posts:key"
+      }
+    ],
+    "/posts/": [
+      {
+        "key": "posts:key:{R:query}",
+        "ttl": "10m"
+      },
+      {
+        "kind": "idempotency",
+        "key": "post-ide-{R:body|hash}",
+        "ttl": "10s",
+        "methods": ["POST"]
       }
     ],
     "/posts/3": [
@@ -49,24 +61,11 @@
         "ttl": "1m",
         "key": "{R:full_path}"
       }
-    ],
-    "/*": [
-      {
-        "kind": "request_id",
-        "methods": ["GET", "POST", "DELETE", "PUT", "PATCH"],
-        "skip_return": true
-      },
-      {
-        "kind": "retry",
-        "methods": ["GET", "PATCH"],
-        "backoff": "1s",
-        "count": 2
-      }
-
     ]
   },
   "options": {
-    "host": "http://localhost:8000",
-    "timeout": "1m30s"
+    "target": "http://localhost:8888",
+    "timeout": "1m30s",
+    "default_storage": "main"
   }
 }

config/config.go

@@ -44,6 +44,54 @@ type Config struct {
 	lock *sync.RWMutex
 }
 
+func (c *Config) Init() error {
+	rules, err := c.getRules()
+	if err != nil {
+		return err
+	}
+	for _, rule := range rules {
+		c.resolver.Add(rule)
+	}
+	return nil
+}
+
+func (c *Config) getRules() ([]*rules.Rule, error) {
+	routes, err := c.repository.GetRoutes()
+	if err != nil {
+		return nil, err
+	}
+	var storage storages.Storage
+	var ok bool
+	defaultStorageName, err := c.repository.GetDefaultStorage()
+	if err != nil {
+		return nil, err
+	}
+	defaultStorage, ok := c.storages[defaultStorageName]
+	if !ok {
+		return nil, errors.New("wrong default Storage setup")
+	}
+	returnRules := []*rules.Rule{}
+
+	for _, route := range routes {
+		rules, err := c.repository.GetRules(route)
+		if err != nil {
+			return nil, err
+		}
+		for _, ruleOptions := range rules {
+			storage, ok = c.storages[ruleOptions.Storage]
+			if !ok {
+				storage = defaultStorage
+			}
+			rule, err := getRuleFromOptions(ruleOptions, storage, route)
+			if err != nil {
+				return nil, err
+			}
+			returnRules = append(returnRules, rule)
+		}
+	}
+	return returnRules, nil
+}
+
 func (c *Config) Resolve(path string) (rules []*rules.Rule, params map[string]string, err error) {
 	return c.resolver.Resolve(path)
 }
@@ -83,6 +131,14 @@ func (c *Config) GetTimeout() (time.Duration, error) {
 	return timeout, nil
 }
 
+func (c *Config) GetStorages() (map[string]string, error) {
+	return c.repository.GetStorages()
+}
+
+func (c *Config) GetRoutes() ([]*rules.Rule, error) {
+	return c.getRules()
+}
+
 func NewConfigFromDSN(dsn string, resolver *resolver.Resolver) (*Config, error) {
 	repo, err := newFileConfig(dsn)
 	if err != nil {
@@ -101,45 +157,11 @@ func NewConfigFromDSN(dsn string, resolver *resolver.Resolver) (*Config, error)
 		}
 		log.Info().Msgf("Configured storage '%v' with dns '%v'", name, DSN)
 	}
-
-	// defRequest := &message.Request{Host: c.Options.Target, Timeout: timeout} // Move to runner
-	routes, err := repo.GetRoutes()
-	if err != nil {
-		return nil, err
-	}
-	var storage storages.Storage
-	var ok bool
-	defaultStorageName, err := repo.GetDefaultStorage()
-	if err != nil {
-		return nil, err
-	}
-	defaultStorage, ok := storagesMap[defaultStorageName]
-	if !ok {
-		return nil, errors.New("wrong default Storage setup")
-	}
-
-	for _, route := range routes {
-		rules, err := repo.GetRules(route)
-		if err != nil {
-			return nil, err
-		}
-		for _, ruleOptions := range rules {
-			storage, ok = storagesMap[ruleOptions.Storage]
-			if !ok {
-				storage = defaultStorage
-			}
-			rule, err := getRuleFromOptions(ruleOptions, storage, route)
-			if err != nil {
-				return nil, err
-			}
-
-			resolver.Add(route, rule)
-		}
-	}
-	return &Config{
+	c := Config{
 		repository: repo,
 		resolver:   resolver,
 		storages:   storagesMap,
 		lock:       &sync.RWMutex{},
-	}, nil
+	}
+	return &c, c.Init()
 }

config/rules.go

@@ -55,6 +55,8 @@ func getRuleProcessorFromOptions(rule Rule) (rules.RuleProcessor, error) {
 		return convertToEarlyCacheRule(rule)
 	case "cache":
 		return convertToCacheRule(rule)
+	case "glue":
+		return convertToGlueRule(rule)
 	case "":
 		return convertToCacheRule(rule)
 	}
@@ -124,6 +126,10 @@ func convertToHitRule(rule Rule) (*rules.HitRule, error) {
 	return &rules.HitRule{TTL: ttl, Hits: rule.Hits, UpdateAfterHits: rule.UpdateAfterHits}, err
 }
 
+func convertToGlueRule(rule Rule) (*rules.GlueRule, error) {
+	return &rules.GlueRule{Calls: rule.Calls}, nil
+}
+
 func timeFromString(in string) (time.Duration, error) {
 	if in == "" {
 		return time.Second, nil