Merge pull request #128 from emqx/develop

Develop

.github/workflows/release.yaml → .github/workflows/build_packages.yaml

@@ -61,11 +61,41 @@ jobs:
           with:
             name: packages-mac
             path: _packages/.
+
+    build-docker-images:
+        runs-on: ubuntu-latest
 
+        steps:
+        - uses: actions/checkout@v2
+        - name: install docker
+          run: |
+            sudo apt-get remove docker docker-engine docker.io containerd runc
+            sudo apt-get update
+            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
+            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+            sudo apt-get update
+            sudo apt-get install docker-ce docker-ce-cli containerd.io
+        - name: prepare docker
+          run: |
+            mkdir -p $HOME/.docker
+            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
+            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | sudo tee /etc/docker/daemon.json
+            sudo systemctl restart docker
+            docker version
+            docker buildx create --use --name mybuild
+        - name: build docker image
+          run: make docker
+        - name: build docker images
+          if: github.event_name == 'release'
+          run: |
+            echo ${{ secrets.DockerHubPassword }} | docker login -u ${{ secrets.DockerHubUser }} --password-stdin
+            make cross_docker
+      
     release:
         runs-on: ubuntu-latest
 
-        needs: [build, build-on-mac]
+        needs: [build, build-on-mac, build-docker-images]
 
         steps:
         - uses: actions/checkout@v2

.github/workflows/fvt_tests.yaml

@@ -0,0 +1,61 @@
+name: Run fvt tests
+
+on:
+    push:
+    pull_request:
+    release:
+        types:
+            - published
+            - prereleased
+
+jobs:
+    fvt_tests:
+        runs-on: ubuntu-latest
+
+        steps:
+        - uses: actions/setup-go@v1
+          with:
+            go-version: '1.11.5'
+        - uses: actions/setup-java@v1
+          with:
+            java-version: '8' # The JDK version to make available on the path.
+            java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+            architecture: x64 # (x64 or x86) - defaults to x64
+        - name: set up jmeter
+          env:
+            JMETER_VERSION: 5.2.1
+          run: |
+            wget -O /tmp/apache-jmeter.tgz http://mirror.bit.edu.cn/apache//jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
+            cd /tmp && tar -xvf apache-jmeter.tgz
+            echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+            echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+            wget -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-1.13-jar-with-dependencies.jar https://github.com/emqx/mqtt-jmeter/raw/master/Download/v1.13.0/mqtt-xmeter-1.13-jar-with-dependencies.jar
+            ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+        - name: install emqx
+          env:
+            EMQX_VERSION: v4.0.2
+          run: |
+            wget -O emqx.deb https://www.emqx.io/downloads/broker/v4.0.2/emqx-ubuntu18.04-${EMQX_VERSION}_amd64.deb
+            sudo dpkg -i emqx.deb
+        - uses: actions/checkout@v2
+        - name: build kuiper
+          run: make
+        - name: run emqx and kuiper
+          run: sudo ./fvt_scripts/setup_env.sh
+        - name: run fvt tests
+          timeout-minutes: 5
+          run: ./fvt_scripts/run_jmeter.sh
+        - uses: actions/upload-artifact@v1
+          with:
+            name: jmeter.logs
+            path: ./jmeter_logs
+        - name: checkout out
+          run: |
+            sudo apt update && sudo apt install -y libxml2-utils
+            cd jmeter_logs
+            if [ "$(xmllint --format --xpath '/testResults/sample/@rc' $(ls *.jtl) | sed -r 's/ /\n/g;' | sort -u | grep -E 'rc=\"[45][0-9][0-9]\"|rc=\"\"')" != "" ]; then
+                echo -e "---------------------------------------------\n"
+                echo "FVT tests error"
+                exit 1
+            fi
+            

.gitignore

@@ -24,4 +24,5 @@ node_modules/
 
 go.sum
 _build
-_packages
+_packages
+jmeter_logs

README-CN.md

@@ -49,9 +49,59 @@ Kuiper 可以运行在各类物联网的边缘使用场景中，比如工业物
 
   提供了与 EMQ X Edge 的无缝集成，实现在边缘端从消息接入到数据分析端到端的场景实现能力
 
+## 快速入门
+
+1. 从 ``https://hub.docker.com/r/emqx/kuiper/tags`` 拉一个 Kuiper 的 docker 镜像。
+
+2. 设置 Kuiper 源为一个 MQTT 服务器。本例使用位于 ``tcp://broker.emqx.io:1883`` 的 MQTT 服务器， ``broker.emqx.io`` 是一个由 [EMQ](https://www.emqx.io) 提供的公有MQTT 服务器。
+
+   ```shell
+   docker run -d --name kuiper -e MQTT_BROKER_ADDRESS=tcp://broker.emqx.io:1883 emqx/kuiper:$tag
+   ```
+
+3. 创建流（stream）- 流式数据的结构定义，类似于数据库中的表格类型定义。比如说要发送温度与湿度的数据到 ``broker.emqx.io``，这些数据将会被在**本地运行的** Kuiper docker 实例中处理。以下的步骤将创建一个名字为 ``demo``的流，并且数据将会被发送至 ``devices/device_001/messages`` 主题，这里的 ``device_001`` 可以是别的设备，比如 ``device_002``，所有的这些数据会被 ``demo`` 流订阅并处理。
+
+   ```shell
+   -- In host
+   # docker exec -it kuiper /bin/sh
+   
+   -- In docker instance
+   # bin/cli create stream demo '(temperature float, humidity bigint) WITH (FORMAT="JSON", DATASOURCE="devices/+/messages")'
+   Connecting to 127.0.0.1:20498...
+   Stream demo is created.
+   
+   # bin/cli query
+   Connecting to 127.0.0.1:20498...
+   kuiper > select * from demo where temperature > 30;
+   Query was submit successfully.
+   
+   ```
+
+4. 您可以使用任何[ MQTT 客户端工具](https://www.emqx.io/cn/blog/mqtt-client-tools)来发布传感器数据到服务器 ``tcp://broker.emqx.io:1883``的主题 ``devices/device_001/messages`` 。以下例子使用 ``mosquitto_pub``。
+
+   ```shell
+   # mosquitto_pub -h broker.emqx.io -m '{"temperature": 40, "humidity" : 20}' -t devices/device_001/messages
+   ```
+
+5. 如果一切顺利的话，您可以看到消息打印在容器的 ``bin/cli query`` 窗口里，请试着发布另外一条``温度``小于30的数据，该数据将会被 SQL 规则过滤掉。
+
+   ```shell
+   kuiper > select * from demo WHERE temperature > 30;
+   [{"temperature": 40, "humidity" : 20}]
+   ```
+
+   如有任何问题，请查看日志文件 ``log/stream.log``。
+
+6. 如果想停止测试，在``bin/cli query``命令行窗口中敲 ``ctrl + c `` ，或者输入 ``exit`` 后回车
+
+7. 想了解更多 EMQ X Kuiper 的功能？请参考以下关于在边缘端使用 EMQ X Kuiper 与 AWS / Azure IoT 云集成的案例。
+
+   - [轻量级边缘计算 EMQ X Kuiper 与 AWS IoT 集成方案](https://www.jianshu.com/p/7c0218fd1ee2)
+   - [轻量级边缘计算 EMQ X Kuiper 与 Azure IoT Hub 集成方案](https://www.jianshu.com/p/49b06751355f) 
+
 ## 性能测试结果
 
-### 测试场景
+### 吞吐量测试支持
 
 - 使用 JMeter MQTT 插件来发送数据到 EMQ X 服务器，消息类似于 ``{"temperature": 10, "humidity" : 90}``， 温度与湿度的值是介于 0 ～ 100 之间的随机整数值
 - Kuiper 从 EMQ X 服务器订阅消息，并且通过 SQL 分析数据： ``SELECT * FROM demo WHERE temperature > 50 `` 
@@ -62,6 +112,21 @@ Kuiper 可以运行在各类物联网的边缘使用场景中，比如工业物
 | 树莓派 3B+                                           | 12k            | sys + user: 70% | 20M  |
 | AWS t2.micro (x86: 1 Core * 1 GB) <br />Ubuntu 18.04 | 10k            | sys + user: 25% | 20M  |
 
+### 最大规则数支持
+
+- 8000 条规则，吞吐量为 800 条消息/秒
+- 配置
+  - AWS 2 核 * 4GB 内存 
+  - Ubuntu
+- 资源消耗
+  - 内存: 89% ~ 72%
+  - CPU: 25%
+  - 400KB - 500KB / 规则
+- 规则
+  - 源: MQTT
+  - SQL: SELECT temperature FROM source WHERE temperature > 20 (90% 数据被过滤) 
+  - 目标: 日志
+
 ## 文档
 
 - [开始使用](docs/zh_CN/getting_started.md) 

README.md

@@ -49,9 +49,59 @@ It can be run at various IoT edge use scenarios, such as real-time processing of
 
   Seamless integration with EMQ X Edge, and provided an end to end solution from messaging to analytics. 
 
+## Quick start
+
+1. Pull a Kuiper Docker image from ``https://hub.docker.com/r/emqx/kuiper/tags``.
+
+2. Set Kuiper source to an MQTT server. This sample uses server locating at ``tcp://broker.emqx.io:1883``. ``broker.emqx.io`` is a public MQTT test server hosted by [EMQ](https://www.emqx.io).
+
+   ```shell
+   docker run -d --name kuiper -e MQTT_BROKER_ADDRESS=tcp://broker.emqx.io:1883 emqx/kuiper:$tag
+   ```
+
+3. Create a stream - the stream is your stream data schema, similar to table definition in database. Let's say the temperature & humidity data are sent to ``broker.emqx.io``, and those data will be processed in your **LOCAL RUN** Kuiper docker instance.  Below steps will create a stream named ``demo``, and data are sent to ``devices/device_001/messages`` topic, while ``device_001`` could be other devices, such as ``device_002``, all of those data will be subscribed and handled by ``demo`` stream.
+
+   ```shell
+   -- In host
+   # docker exec -it kuiper /bin/sh
+   
+   -- In docker instance
+   # bin/cli create stream demo '(temperature float, humidity bigint) WITH (FORMAT="JSON", DATASOURCE="devices/+/messages")'
+   Connecting to 127.0.0.1:20498...
+   Stream demo is created.
+   
+   # bin/cli query
+   Connecting to 127.0.0.1:20498...
+   kuiper > select * from demo where temperature > 30;
+   Query was submit successfully.
+   
+   ```
+
+4. Publish sensor data to topic ``devices/device_001/messages`` of server ``tcp://broker.emqx.io:1883`` with any [MQTT client tools](https://medium.com/@emqtt/mqtt-client-tools-215ff7a17ad). Below sample uses ``mosquitto_pub``. 
+
+   ```shell
+   # mosquitto_pub -h broker.emqx.io -m '{"temperature": 40, "humidity" : 20}' -t devices/device_001/messages
+   ```
+
+5. If everything goes well,  you can see the message is print on docker ``bin/cli query`` window. Please try to publish another message with ``temperature`` less than 30, and it will be filtered by WHERE condition of the SQL. 
+
+   ```
+   kuiper > select * from demo WHERE temperature > 30;
+   [{"temperature": 40, "humidity" : 20}]
+   ```
+
+   If having any problems, please take a look at ``log/stream.log``.
+
+6. To stop the test, just press ``ctrl + c `` in ``bin/cli query`` command console, or input `exit` and press enter.
+
+7. Next for exploring more powerful features of EMQ X  Kuiper? Refer to below for how to apply EMQ X Kuiper in edge and integrate with AWS / Azure IoT cloud.
+
+   - [Lightweight edge computing EMQ X Kuiper and Azure IoT Hub integration solution](https://www.emqx.io/blog/85) 
+   - [Lightweight edge computing EMQ X Kuiper and AWS IoT Hub integration solution](https://www.emqx.io/blog/88)
+
 ## Performance test result
 
-### Test scenario
+### Throughput test
 
 - Using JMeter MQTT plugin to send simulation data to EMQ X Broker, such as: ``{"temperature": 10, "humidity" : 90}``, the value of temperature and humidity are random integer between 0 - 100.
 - Kuiper subscribe from EMQ X Broker, and analyze data with SQL: ``SELECT * FROM demo WHERE temperature > 50 `` 
@@ -62,6 +112,21 @@ It can be run at various IoT edge use scenarios, such as real-time processing of
 | Raspberry Pi 3B+                               | 12k                  | sys+user: 70% | 20M          |
 | AWS t2.micro( 1 Core * 1 GB) <br />Ubuntu18.04 | 10k                  | sys+user: 25% | 20M          |
 
+### Max support rule support
+
+- 8000 rules with 800 message/second
+- Configurations
+  - 2 core * 4GB memory in AWS
+  - Ubuntu
+- Resource usage
+  - Memory: 89% ~ 72%
+  - CPU: 25%
+  - 400KB - 500KB / rule
+- Rule
+  - Source: MQTT
+  - SQL: SELECT temperature FROM source WHERE temperature > 20 (90% data are filtered) 
+  - Sink: Log
+
 ## Documents
 
 - [Getting started](docs/en_US/getting_started.md)