Merge pull request #1564 from kubroid/istio-tcp-canary

Istio Canary TCP service support

Author: stefanprodan

Makefile

@@ -29,6 +29,11 @@ test-codegen:
 test: test-fmt test-codegen
 	go test ./...
 
+test-coverage: test-fmt test-codegen
+	go test -coverprofile cover.out ./...
+	go tool cover -html=cover.out
+	rm cover.out
+
 crd:
 	cat artifacts/flagger/crd.yaml > charts/flagger/crds/crd.yaml
 	cat artifacts/flagger/crd.yaml > kustomize/base/flagger/crd.yaml

docs/gitbook/tutorials/istio-progressive-delivery.md

@@ -480,3 +480,63 @@ With the above configuration, Flagger will run a canary release with the followi
 
 The above procedure can be extended with [custom metrics](../usage/metrics.md) checks, [webhooks](../usage/webhooks.md), [manual promotion](../usage/webhooks.md#manual-gating) approval and [Slack or MS Teams](../usage/alerting.md) notifications.
 
+
+## Canary Deployments for TCP Services
+
+Performing a Canary deployment on a TCP (non HTTP) service is nearly identical to an HTTP Canary. Besides updating your `Gateway` document to support the `TCP` routing, the only difference is you have to set the `appProtocol` field to `TCP` inside of the `service` section of your `Canary` document.
+
+#### Example:
+
+```yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: public-gateway
+  namespace: istio-system
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+    - port:
+        number: 7070
+        name: tcp-service
+        protocol: TCP # <== set the protocol to tcp here
+      hosts:
+        - "*"
+```
+
+```yaml
+apiVersion: flagger.app/v1beta1
+kind: Canary
+...
+...
+service:
+  port: 7070
+  appProtocol: TCP # <== set the appProtocol here
+  targetPort: 7070
+  portName: "tcp-service-port"
+...
+...
+```
+
+If the `appProtocol` equals `TCP` then Flagger will treat this as a Canary deployment for a `TCP` service. When it creates the `VirtualService` document it will add a `TCP` section to route requests between the `primary` and `canary` services. See Istio documentation for more information on this [spec](https://istio.io/latest/docs/reference/config/networking/virtual-service/#TCPRoute).
+
+The resulting `VirtualService` will include a `tcp` section similar to what is shown below:
+```yaml
+tcp:
+  - route:
+    - destination:
+        host: tcp-service-primary
+        port:
+          number: 7070
+      weight: 100
+    - destination:
+        host: tcp-service-canary
+        port:
+          number: 7070
+      weight: 0
+```
+
+Once the Canary analysis begins, Flagger will be able to adjust the weights inside of this `tcp` section to advance the Canary deployment until it either runs into an error (and is halted) or it successfully reaches the end of the analysis and is Promoted.
+
+It is also important to note that if you set `appProtocol` to anything other than `TCP`, for example if you set it to `HTTP`, it will perform the Canary and treat it as an `HTTP` service. The same remains true if you do not set `appProtocol` at all. It will __ONLY__ treat a Canary as a `TCP` service if `appProtocal` equals `TCP`.

pkg/apis/istio/v1alpha3/virtual_service.go

@@ -597,7 +597,7 @@ type TCPRoute struct {
 	// Currently, only one destination is allowed for TCP services. When TCP
 	// weighted routing support is introduced in Envoy, multiple destinations
 	// with weights can be specified.
-	Route HTTPRouteDestination `json:"route"`
+	Route []HTTPRouteDestination `json:"route"`
 }
 
 // L4 connection match attributes. Note that L4 connection matching support

pkg/apis/istio/v1alpha3/zz_generated.deepcopy.go

@@ -125,6 +125,23 @@ func (ir *IstioRouter) reconcileDestinationRule(canary *flaggerv1.Canary, name s
 	return nil
 }
 
+// return true if canary service has appProtocol == tcp
+func isTcp(canary *flaggerv1.Canary) bool {
+	return strings.ToLower(canary.Spec.Service.AppProtocol) == "tcp"
+}
+
+// map canary.spec.service.match into L4Match
+func canaryToL4Match(canary *flaggerv1.Canary) []istiov1alpha3.L4MatchAttributes {
+	var match []istiov1alpha3.L4MatchAttributes
+	for _, m := range canary.Spec.Service.Match {
+		match = append(match, istiov1alpha3.L4MatchAttributes{
+			Port: int(m.Port),
+		})
+	}
+
+	return match
+}
+
 func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 	apexName, primaryName, canaryName := canary.GetServiceNames()
 
@@ -175,20 +192,35 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 		gateways = []string{}
 	}
 
-	newSpec := istiov1alpha3.VirtualServiceSpec{
-		Hosts:    hosts,
-		Gateways: gateways,
-		Http: []istiov1alpha3.HTTPRoute{
-			{
-				Match:      canary.Spec.Service.Match,
-				Rewrite:    canary.Spec.Service.GetIstioRewrite(),
-				Timeout:    canary.Spec.Service.Timeout,
-				Retries:    canary.Spec.Service.Retries,
-				CorsPolicy: canary.Spec.Service.CorsPolicy,
-				Headers:    canary.Spec.Service.Headers,
-				Route:      canaryRoute,
+	var newSpec istiov1alpha3.VirtualServiceSpec
+
+	if isTcp(canary) {
+		newSpec = istiov1alpha3.VirtualServiceSpec{
+			Hosts:    hosts,
+			Gateways: gateways,
+			Tcp: []istiov1alpha3.TCPRoute{
+				{
+					Match: canaryToL4Match(canary),
+					Route: canaryRoute,
+				},
 			},
-		},
+		}
+	} else {
+		newSpec = istiov1alpha3.VirtualServiceSpec{
+			Hosts:    hosts,
+			Gateways: gateways,
+			Http: []istiov1alpha3.HTTPRoute{
+				{
+					Match:      canary.Spec.Service.Match,
+					Rewrite:    canary.Spec.Service.GetIstioRewrite(),
+					Timeout:    canary.Spec.Service.Timeout,
+					Retries:    canary.Spec.Service.Retries,
+					CorsPolicy: canary.Spec.Service.CorsPolicy,
+					Headers:    canary.Spec.Service.Headers,
+					Route:      canaryRoute,
+				},
+			},
+		}
 	}
 
 	newMetadata := canary.Spec.Service.Apex
@@ -203,7 +235,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 	}
 	newMetadata.Annotations = filterMetadata(newMetadata.Annotations)
 
-	if len(canary.GetAnalysis().Match) > 0 {
+	if !isTcp(canary) && len(canary.GetAnalysis().Match) > 0 {
 		canaryMatch := mergeMatchConditions(canary.GetAnalysis().Match, canary.Spec.Service.Match)
 		newSpec.Http = []istiov1alpha3.HTTPRoute{
 			{
@@ -349,6 +381,38 @@ func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
 		return
 	}
 
+	if isTcp(canary) {
+		ir.logger.Infof("Canary %s.%s uses TCP service", canary.Name, canary.Namespace)
+		var tcpRoute istiov1alpha3.TCPRoute
+		for _, tcp := range vs.Spec.Tcp {
+			for _, r := range tcp.Route {
+				if r.Destination.Host == canaryName {
+					tcpRoute = tcp
+					break
+				}
+			}
+		}
+		for _, route := range tcpRoute.Route {
+			if route.Destination.Host == primaryName {
+				primaryWeight = route.Weight
+			}
+			if route.Destination.Host == canaryName {
+				canaryWeight = route.Weight
+			}
+		}
+
+		mirrored = false
+
+		if primaryWeight == 0 && canaryWeight == 0 {
+			err = fmt.Errorf("VirtualService %s.%s does not contain routes for %s-primary and %s-canary",
+				apexName, canary.Namespace, apexName, apexName)
+		}
+
+		return
+	}
+
+	ir.logger.Infof("Canary %s.%s uses HTTP service", canary.Name, canary.Namespace)
+
 	var httpRoute istiov1alpha3.HTTPRoute
 	for _, http := range vs.Spec.Http {
 		for _, r := range http.Route {
@@ -412,6 +476,26 @@ func (ir *IstioRouter) SetRoutes(
 
 	vsCopy := vs.DeepCopy()
 
+	if isTcp(canary) {
+		// weighted routing (progressive canary)
+		weightedRoute := istiov1alpha3.TCPRoute{
+			Match: canaryToL4Match(canary),
+			Route: []istiov1alpha3.HTTPRouteDestination{
+				makeDestination(canary, primaryName, primaryWeight),
+				makeDestination(canary, canaryName, canaryWeight),
+			},
+		}
+		vsCopy.Spec.Tcp = []istiov1alpha3.TCPRoute{
+			weightedRoute,
+		}
+
+		vs, err = ir.istioClient.NetworkingV1alpha3().VirtualServices(canary.Namespace).Update(context.TODO(), vsCopy, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("VirtualService %s.%s update failed: %w", apexName, canary.Namespace, err)
+		}
+		return nil
+	}
+
 	// weighted routing (progressive canary)
 	weightedRoute := istiov1alpha3.HTTPRoute{
 		Match:      canary.Spec.Service.Match,