Bug fixes and added manifests with services for all data processor pr…

samples/anomaly-detection/Dockerfile

@@ -14,8 +14,8 @@ RUN go install github.com/magefile/mage@latest
 
 RUN mage ci
 
-RUN go build -o ./bin/anomaly-detection ./cmd/anomaly-detection
+RUN go build -o ./bin/anomaly-detection ./cmd
 
 EXPOSE 2112
 
-CMD [ "/bin/anomaly-detection" ]
+CMD [ "./bin/anomaly-detection" ]

samples/anomaly-detection/README.md

@@ -2,21 +2,23 @@
 
 ## Usage
 
+This demo assumes a K3S cluster.
+
 ### Server as a Pod
 
 ```sh
 # From the root of the anomaly-detection directory.
-docker build ../.. -f Dockerfile -t <yourcrhere>.azurecr.io/anomaly-detection:latest
+docker build ../.. -f Dockerfile -t makocr.azurecr.io/anomaly-detection:latest
 
 # Or if running from the root of the explore-iot-operations repository.
-# docker build . -f ./samples/anomaly-detection/Dockerfile -t <yourcrhere>.azurecr.io/anomaly-detection:latest
+# docker build . -f ./samples/anomaly-detection/Dockerfile -t makocr.azurecr.io/anomaly-detection:latest
 
 # Push or load your newly built image into your cluster, depending on the k8s setup.
-# docker push <yourcrhere>.azurecr.io/anomaly-detection:latest # Using AKS + Connected ACR
-# minikube load <yourcrhere>.azurecr.io/anomaly-detection:latest # Using minikube
-# docker save <yourcrhere>.azurecr.io/anomaly-detection:latest | k3s ctr images import - # Using K3s
+# docker push makocr.azurecr.io/anomaly-detection:latest # Using AKS + Connected ACR
+# minikube load makocr.azurecr.io/anomaly-detection:latest # Using minikube
+# docker save makocr.azurecr.io/anomaly-detection:latest | k3s ctr images import - # Using K3s
 
-kubectl run anomaly-detection --image=<yourcrhere>.azurecr.io/anomaly-detection:latest --stdin < config.yml
+kubectl apply -f manifest.yml
 ```
 
 ### Configuration

samples/anomaly-detection/go.mod

@@ -12,21 +12,27 @@ replace (
 require (
 	github.com/explore-iot-ops/lib/env v0.0.0-00010101000000-000000000000
 	github.com/explore-iot-ops/lib/logger v0.0.0-00010101000000-000000000000
+	github.com/explore-iot-ops/lib/mage v0.0.0-00010101000000-000000000000
 	github.com/gofiber/fiber/v2 v2.51.0
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/VividCortex/ewma v1.1.1 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/cheggaaa/pb/v3 v3.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/color v1.9.0 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/princjef/mageutil v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.50.0 // indirect

samples/anomaly-detection/go.sum

@@ -1,26 +1,47 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/cheggaaa/pb v2.0.7+incompatible/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
+github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM=
+github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
 github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
+github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/matryer/is v1.3.0 h1:9qiso3jaJrOe6qBRJRBt2Ldht05qDiFP9le0JOIhRSI=
+github.com/matryer/is v1.3.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/princjef/mageutil v1.0.0 h1:1OfZcJUMsooPqieOz2ooLjI+uHUo618pdaJsbCXcFjQ=
+github.com/princjef/mageutil v1.0.0/go.mod h1:mkShhaUomCYfAoVvTKRcbAs8YSVPdtezI5j6K+VXhrs=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
@@ -34,12 +55,23 @@ github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e
 github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v2 v2.0.7/go.mod h1:0CiZ1p8pvtxBlQpLXkHuUTpdJ1shm3OqCF1QugkjHL4=
+gopkg.in/fatih/color.v1 v1.7.0/go.mod h1:P7yosIhqIl/sX8J8UypY5M+dDpD2KmyfP5IRs5v/fo0=
+gopkg.in/mattn/go-colorable.v0 v0.1.0/go.mod h1:BVJlBXzARQxdi3nZo6f6bnl5yR20/tOL6p+V0KejgSY=
+gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc=
+gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

samples/anomaly-detection/lib/ewma/emwa_test.go

@@ -67,7 +67,6 @@ func TestSquareSum(t *testing.T) {
 	nextMean = ewma.Mean(5, obs, mean)
 	sum = ewma.SquareSum(obs, sum, nextMean, mean)
 	require.Equal(t, 65.2, sum)
-	mean = nextMean
 }
 
 func TestControlLimit(t *testing.T) {

samples/anomaly-detection/magefile.go

@@ -13,7 +13,11 @@ import (
 func CI() error {
 	return mage.CI(
 		"github.com/explore-iot-ops/samples/anomaly-detection/",
-		map[string]any{"cmd": nil},
+		map[string]any{
+			"cmd":               nil,
+			"lib/configuration": nil,
+			"lib/payload":       nil,
+		},
 		3000,
 		0.00,
 		0.00,

samples/anomaly-detection/manifest.yml

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: azure-iot-operations
+  name: anomaly-conf
+data:
+  config.yml: |-
+      logger:
+        level: 0
+      server:
+        route: /anomaly
+        port: 3333
+      algorithm:
+        temperature:
+          lambda: 0.25
+          lFactor: 3
+          controlT: 90
+          controlS: 20
+          controlN: 10
+        vibration:
+          lambda: 0.25
+          lFactor: 3
+          controlT: 50
+          controlS: 20
+          controlN: 10
+        humidity:
+          lambda: 0.25
+          lFactor: 3
+          controlT: 80
+          controlS: 20
+          controlN: 10
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: azure-iot-operations
+  name: anomaly-detection
+  labels:
+    app: anomaly-detection
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: anomaly-detection
+  template:
+    metadata:
+      labels:
+        app: anomaly-detection
+    spec:
+      containers:
+        - name: anomaly-detection
+          image: makocr.azurecr.io/anomaly-detection:latest
+          imagePullPolicy: Always
+          ports:
+          - name: http
+            containerPort: 3333
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "250m"
+            limits:
+              memory: "128Mi"
+              cpu: "500m"
+          command:
+            - "./bin/anomaly-detection"
+            - "--stdin=false"
+            - "--config=/etc/anomaly-detection/config.yml"
+          volumeMounts:
+            - mountPath: /etc/anomaly-detection/config.yml
+              subPath: config.yml
+              name: config
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: anomaly-conf
+---
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: azure-iot-operations
+  name: anomaly-svc
+  labels:
+    app: anomaly-detection
+spec:
+  ports:
+  - port: 3333
+    name: http
+  type: ClusterIP
+  selector:
+    app: anomaly-detection

samples/http-grpc-callout/Dockerfile

@@ -14,8 +14,8 @@ RUN go install github.com/magefile/mage@latest
 
 RUN mage ci
 
-RUN go build -o ./bin/http-grpc-callout ./cmd/http-grpc-callout
+RUN go build -o ./bin/http-grpc-callout ./cmd
 
 EXPOSE 2112
 
-CMD [ "/bin/http-grpc-callout" ]
+CMD [ "./bin/http-grpc-callout" ]

samples/http-grpc-callout/README.md

@@ -8,17 +8,17 @@ GRPC/HTTP Callout Server is a server for testing the http and grpc callout capab
 
 ```sh
 # From the root of the http-grpc-callout directory.
-docker build ../.. -f Dockerfile -t <yourcrhere>.azurecr.io/http-grpc-callout:latest
+docker build ../.. -f Dockerfile -t makocr.azurecr.io/http-grpc-callout:latest
 
 # Or if running from the root of the explore-iot-operations repository.
-# docker build . -f ./samples/http-grpc-callout/Dockerfile -t <yourcrhere>.azurecr.io/http-grpc-callout:latest
+# docker build . -f ./samples/http-grpc-callout/Dockerfile -t makocr.azurecr.io/http-grpc-callout:latest
 
 # Push or load your newly built image into your cluster, depending on the k8s setup.
-# docker push <yourcrhere>.azurecr.io/http-grpc-callout:latest # Using AKS + Connected ACR
-# minikube load <yourcrhere>.azurecr.io/http-grpc-callout:latest # Using minikube
-# docker save <yourcrhere>.azurecr.io/http-grpc-callout:latest | k3s ctr images import - # Using K3s
+# docker push makocr.azurecr.io/http-grpc-callout:latest # Using AKS + Connected ACR
+# minikube load makocr.azurecr.io/http-grpc-callout:latest # Using minikube
+# docker save makocr.azurecr.io/http-grpc-callout:latest | k3s ctr images import - # Using K3s
 
-kubectl run http-grpc-callout --image=<yourcrhere>.azurecr.io/http-grpc-callout:latest --stdin < config.yml
+kubectl apply -f manifest.yml
 ```
 
 ### Configuration