ci.yml

name: Integration Test WF
on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
env:
  KIND_URL: https://kind.sigs.k8s.io/dl
  KIND_VER: v0.8.1
  KIND_ENV: kind-linux-amd64

jobs:
  check:
    runs-on: ubuntu-latest
    outputs:
      packages: ${{ steps.filter.outputs.changes }}
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
        id: filter
        with:
          filters: |
            kafkachanges:
              - 'kafka-http-connector/**'
            # rabbitmqchanges:
            #   - 'rabbitmq-http-connector/**'
            # sqschanges:
            #   - 'aws-sqs-http-connector/**'
            # redischanges:
            #   - 'redis-http-connector/**'
            # natschanges:
            #   - 'nats-streaming-http-connector/**'
            # natsjetstreamchanges:
            #   - 'nats-jetstream-http-connector/**'

  kafka:
    needs: check
    if: contains(needs.check.outputs.packages, 'kafkachanges')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Setup Keda
        uses: ./.github/actions/setup-keda

      - name: Deploy Kafka Cluster
        run: |
          kubectl create namespace kafka
          curl -L http://strimzi.io/install/latest | sed 's/namespace: .*/namespace: kafka/' | kubectl create -f - -n kafka
          sleep 10s
          kubectl create -f kafka-http-connector/test/kubernetes/kafka-cluster.yaml
          echo "Kafka Cluster is getting up."
          kubectl wait -f kafka-http-connector/test/kubernetes/kafka-cluster.yaml --for=condition=ready --timeout=-1s -n kafka
          sleep 10s
          kubectl get pods -n kafka
          kubectl wait pod -l app.kubernetes.io/name=zookeeper --for=condition=ready --timeout=-1s -n kafka

      - name: Create Kafka topics
        run: |
          kubectl apply -f kafka-http-connector/test/kubernetes/kafka-req-topic.yaml
          kubectl apply -f kafka-http-connector/test/kubernetes/kafka-err-topic.yaml
          kubectl apply -f kafka-http-connector/test/kubernetes/kafka-res-topic.yaml

      - name: Deploy Kafka Keda Connector and Keda ScaledObject
        run: |
          ko resolve -f kafka-http-connector/test/kubernetes/keda-deployment.yml | kubectl apply -f - -n kafka
          kubectl get pods -n kafka
          sleep 15s
          kubectl get pods -n kafka
          kubectl wait pod -l keda=kafka --for=condition=ready --timeout=30s -n kafka
          kubectl apply -f kafka-http-connector/test/kubernetes/Keda-ScaledObj.yml -n kafka

      - name: Produce Kafka messages Using Producer
        run: |
          ko resolve -f kafka-http-connector/test/kubernetes/kafka-produer.yaml | kubectl apply -f - -n kafka
          kubectl wait job -l app=pi --for=condition=complete --timeout=-1s -n kafka
          kubectl delete job pi -n kafka

      - name: Collect Kafka Consumer Messages
        run: |
          kubectl get pods -n kafka
          ko resolve -f kafka-http-connector/test/consumer/consumer-deployment.yaml | kubectl apply -f - -n kafka
          kubectl wait pod -l app=consumer --for=condition=ready --timeout=-1s -n kafka
          kubectl logs -l app=consumer --all-containers=true -n kafka

  rabbitmq:
    needs: check
    if: contains(needs.check.outputs.packages, 'rabbitmqchanges')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Setup Keda
        uses: ./.github/actions/setup-keda

      - name: Create Docker Image for Rabbitmq KEDA Connector
        run: |
          cd rabbitmq-http-connector/
          docker build -t localhost:5000/rabbit-keda:latest .
          docker push localhost:5000/rabbit-keda:latest
      - name: Create Rabbitmq Docker Image for Publisher
        run: |
          cd rabbitmq-http-connector/test/publisher/
          docker build -t localhost:5000/rabbit-publisher:latest .
          docker push localhost:5000/rabbit-publisher:latest
      - name: Create Rabbitmq Docker Image for Consumer
        run: |
          cd rabbitmq-http-connector/test/consumer/
          docker build -t localhost:5000/rabbit-consumer:latest .
          docker push localhost:5000/rabbit-consumer:latest
      - name: Deploy Rabbitmq Deployment files
        run: |
          cd rabbitmq-http-connector/test/kubernetes/
          kubectl create ns rabbits
          kubectl apply -n rabbits -f rabbit-rbac.yaml
          kubectl apply -n rabbits -f rabbit-configmap.yaml
          kubectl apply -n rabbits -f rabbit-secret.yaml
          kubectl apply -n rabbits -f rabbit-statefulset.yaml
          bash rabbitmq-cluster-instances.sh
      - name: Publish Rabbitmq messages in a queue
        run: |
          cd rabbitmq-http-connector/test/publisher/
          kubectl apply -f deployment.yaml
          kubectl apply -f publisher-job.yaml
      - name: Bring up the Rabbitmq consumer queue and Listen for incoming messages
        run: |
          cd rabbitmq-http-connector/test/consumer/
          kubectl apply -f consumer-deployment.yaml
      - name: Deploy Rabbitmq Keda Connector and Keda ScaledObject
        run: |
          cd rabbitmq-http-connector/test/kubernetes/
          kubectl apply -f keda-deployment.yml
          kubectl wait pod -l keda=rabbitmq --for=condition=ready --timeout=-1s -n rabbits
          kubectl apply -f Keda-ScaledObj.yml
      - name: Get Rabbitmq consumed messages from queue
        run: |
          sleep 10s
          kubectl logs  -n rabbits deployment.apps/rabbitmq-consumer
  sqs:
    needs: check
    if: contains(needs.check.outputs.packages, 'sqschanges')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Setup Keda
        uses: ./.github/actions/setup-keda

      - name: Create Docker Image for SQS KEDA Connector
        run: |
          cd aws-sqs-http-connector
          docker build -t localhost:5000/aws-sqs-connector:latest .
          docker push localhost:5000/aws-sqs-connector:latest
      - name: Create SQS Docker image with AWS CLI in KEDA Connetor
        run: |
          cd aws-sqs-http-connector/test/kubernetes
          docker build -t localhost:5000/aws-sqs-test-connector:latest .
          docker push localhost:5000/aws-sqs-test-connector:latest
      - name: Deploy SQS Keda Connector, Keda ScaledObject and Localstack
        run: |
          cd aws-sqs-http-connector/test/kubernetes
          kubectl apply -f keda-deployment.yml
          kubectl apply -f keda-scaledObj.yml
          kubectl apply -f localstack.yaml
      - name: Build and Deploy SQS test queue
        run: |
          cd aws-sqs-http-connector/test/validate
          docker build -t localhost:5000/test-queue:latest .
          docker push localhost:5000/test-queue:latest
          kubectl apply -f test.yaml
          sleep 10s
          kubectl get pods
          kubectl wait pod -l app=queue --for=condition=ready --timeout=200s
      - name: Get SQS consumed messages from queue
        run: |
          sleep 30s
          kubectl logs -l app=queue
  redis:
    needs: check
    if: contains(needs.check.outputs.packages, 'redischanges')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Setup Keda
        uses: ./.github/actions/setup-keda

      - name: Create Docker Image for Redis KEDA Connector
        run: |
          cd redis-http-connector/
          docker build -t localhost:5000/redis-connector:latest .
          docker push localhost:5000/redis-connector:latest
      - name: Create Redis Docker Image for Publisher
        run: |
          cd redis-http-connector/test/publisher/
          docker build -t localhost:5000/redis-publisher:latest .
          docker push localhost:5000/redis-publisher:latest
      - name: Create Redis Docker Image for Consumer
        run: |
          cd redis-http-connector/test/consumer/
          docker build -t localhost:5000/redis-consumer:latest .
          docker push localhost:5000/redis-consumer:latest
      - name: Deploy Redis cluster
        run: |
          helm repo add ot-helm https://ot-container-kit.github.io/helm-charts/
          kubectl create namespace redis-operator
          helm upgrade redis-operator ot-helm/redis-operator --install --namespace redis-operator
          sleep 50s
          cd redis-http-connector/test/kubernetes
          kubectl create namespace ot-operators
          helm upgrade redis ot-helm/redis -f redis-values.yaml --install --namespace ot-operators
          sleep 50s
          kubectl get pods -n ot-operators
      - name: Deploy Redis Keda Connector and Keda ScaledObject
        run: |
          cd redis-http-connector/test/kubernetes/
          kubectl apply -f keda-deployment.yaml
          sleep 30s
          kubectl apply -f Keda-ScaledObj.yaml
          sleep 30s
      - name: Publish Redis messages in a queue
        run: |
          cd redis-http-connector/test/publisher/
          kubectl apply -f publisher-deployment.yaml
          sleep 30s
      - name: Bring up the Redis consumer queue and Listen for incoming messages
        run: |
          cd redis-http-connector/test/consumer/
          kubectl apply -f consumer-deployment.yaml
      - name: Get Redis consumed messages from queue
        run: |
          sleep 30s
          kubectl logs deployment.apps/consumer-deployment
          kubectl logs deployment.apps/consumer-deployment | grep -q "consumed"
          echo $?
  nats:
    needs: check
    if: contains(needs.check.outputs.packages, 'natschanges')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Setup Keda
        uses: ./.github/actions/setup-keda

      - name: Create Docker Image for Nats KEDA Connector
        run: |
          cd nats-streaming-http-connector/
          docker build -t localhost:5000/nats-connector:latest .
          docker push localhost:5000/nats-connector:latest
      - name: Create Nats Docker Image for Publisher
        run: |
          cd nats-streaming-http-connector/test/producer/
          docker build -t localhost:5000/nats-producer:latest .
          docker push localhost:5000/nats-producer:latest
      - name: Create Nats Docker Image for Consumer
        run: |
          cd nats-streaming-http-connector/test/consumer/
          docker build -t localhost:5000/nats-consumer:latest .
          docker push localhost:5000/nats-consumer:latest
      - name: Deploy Nats streaming cluster
        run: |
          helm repo add nats https://nats-io.github.io/k8s/helm/charts/
          helm repo update
          helm install nats nats/nats
          sleep 50s
          helm install my-stan nats/stan --set stan.nats.url=nats://nats:4222
          sleep 30s
          kubectl get pods
      - name: Deploy Nats Keda Connector and Keda ScaledObject
        run: |
          cd nats-streaming-http-connector/test/kubernetes/
          kubectl apply -f Keda-deployment.yaml
          sleep 30s
          kubectl apply -f keda-ScaledObject.yaml
          sleep 30s
      - name: Publish Nats messages in a queue
        run: |
          cd nats-streaming-http-connector/test/producer/
          kubectl apply -f deployment.yaml
          sleep 30s
      - name: Bring up the Nats consumer queue and Listen for incoming messages
        run: |
          cd nats-streaming-http-connector/test/consumer/
          kubectl apply -f deployment.yaml
      - name: Get Nats consumed messages from queue
        run: |
          sleep 30s
          kubectl logs deployment.apps/consumer-deployment
          kubectl logs deployment.apps/consumer-deployment | grep -q "consumed"
          echo $?

  natsjetstream:
    needs: check
    if: contains(needs.check.outputs.packages, 'natsjetstreamchanges')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Setup Keda
        uses: ./.github/actions/setup-keda

      - name: Create Docker Image for Jetstream Nats KEDA Connector
        run: |
          cd nats-jetstream-http-connector/
          docker build -t localhost:5000/jetstream-connector:latest .
          docker push localhost:5000/jetstream-connector:latest
      - name: Create Nats Docker Image for Publisher
        run: |
          cd nats-jetstream-http-connector/test/producer/
          docker build -t localhost:5000/jetstream-producer:latest .
          docker push localhost:5000/jetstream-producer:latest
      - name: Create Nats Docker Image for Consumer
        run: |
          cd nats-jetstream-http-connector/test/consumer/
          docker build -t localhost:5000/jetstream-consumer:latest .
          docker push localhost:5000/jetstream-consumer:latest
      - name: Deploy Jetstream Nats cluster
        run: |
          cd nats-jetstream-http-connector/test/jetstream
          kubectl apply -f jetstream-server.yaml
      - name: Publish Nats messages in a stream
        run: |
          cd nats-jetstream-http-connector/test/producer/
          kubectl apply -f deployment.yaml
          sleep 30s
      - name: Bring up the Nats consumer stream and Listen for incoming messages
        run: |
          cd nats-jetstream-http-connector/test/consumer/
          kubectl apply -f deployment.yaml
      - name: Deploy Nats Keda Connector and Keda ScaledObject
        run: |
          cd nats-jetstream-http-connector/test/kubernetes/
          kubectl apply -f Keda-deployment.yaml
          sleep 30s
          kubectl apply -f keda-ScaledObject.yaml
          sleep 30s
      - name: Get Nats consumed messages from stream
        run: |
          # sleep 90s
          kubectl wait pod -l app=consumer --for=condition=ready --timeout=-1s
          kubectl logs deployment.apps/consumer-deployment | grep -q "consumed"
          echo $?