[CI] Add binary run tests

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: OctoBot-Binary-CI
 on: push
 
 jobs:
-  builds:
+  build:
     name: ${{ matrix.os }} - ${{ matrix.arch }} - Python 3.8 - build
     runs-on: ${{ matrix.os }}
     strategy:
@@ -59,7 +59,7 @@ jobs:
       env:
         GH_REPO: Drakkar-Software/OctoBot-Tentacles
         OCTOBOT_GH_REPO: https://github.com/Drakkar-Software/OctoBot.git
-        OCTOBOT_DEFAULT_BRANCH: dev
+        OCTOBOT_DEFAULT_BRANCH: master
         OCTOBOT_REPOSITORY_DIR: OctoBot
         NLTK_DATA: nltk_data
         BUILD_ARCH: ${{ matrix.arch }}
@@ -70,7 +70,7 @@ jobs:
       env:
         GH_REPO: Drakkar-Software/OctoBot-Tentacles
         OCTOBOT_GH_REPO: https://github.com/Drakkar-Software/OctoBot.git
-        OCTOBOT_DEFAULT_BRANCH: dev
+        OCTOBOT_DEFAULT_BRANCH: master
         OCTOBOT_REPOSITORY_DIR: OctoBot
         NLTK_DATA: nltk_data
       run: .\build_scripts\windows.ps1
@@ -90,10 +90,72 @@ jobs:
         path: OctoBot/dist/OctoBot_windows.exe
         if-no-files-found: error
 
+  test:
+    name: ${{ matrix.os }} - ${{ matrix.arch }} - Python 3.8 - test
+    needs:
+      - build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ macos-latest, windows-latest, ubuntu-latest ]
+        arch: [ x64 ]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8.x'
+          architecture: ${{ matrix.arch }}
+
+      - name: Install dependencies
+        run: pip install -r dev_requirements.txt -r requirements.txt
+
+      - name: Download Windows x64 artifact
+        if: matrix.os == 'windows-latest'
+        uses: actions/download-artifact@v2
+        with:
+          name: OctoBot_windows_x64
+          path: OctoBot_windows_x64.exe
+
+      - name: Test OctoBot Binary on Windows
+        if: matrix.os == 'windows-latest'
+        run: |
+          pytest tests --full-trace
+
+      - name: Download Linux x64 artifact
+        if: matrix.os == 'ubuntu-latest'
+        uses: actions/download-artifact@v2
+        with:
+          name: OctoBot_ubuntu-latest_x64
+          path: OctoBot_linux_x64
+
+      - name: Test OctoBot Binary on Linux
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          chmod +x OctoBot_linux_x64/OctoBot_x64
+          pytest tests
+
+      - name: Download MacOs x64 artifact
+        if: matrix.os == 'macos-latest'
+        uses: actions/download-artifact@v2
+        with:
+          name: OctoBot_macos-latest_x64
+          path: OctoBot_macos_x64
+
+      - name: Test OctoBot Binary on MacOs
+        if: matrix.os == 'macos-latest'
+        run: |
+          chmod +x OctoBot_macos_x64/OctoBot_x64
+          pytest tests
+
   create-release:
     name: Create Release
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-    needs: builds
+    needs:
+      - build
+      - test
     runs-on: ubuntu-latest
     outputs:
       release-url-output: ${{ steps.create_release.outputs.upload_url }}
@@ -199,7 +261,8 @@ jobs:
     name: Notify
     runs-on: ubuntu-latest
     needs:
-      - builds
+      - build
+      - test
     if: ${{ failure() }}
 
     steps:

.gitignore

@@ -97,4 +97,7 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-\.idea/
+.idea
+tentacles
+user
+logs

build_scripts/windows.ps1

@@ -6,7 +6,7 @@ python scripts/python_file_lister.py bin/octobot_packages_files.txt $env:OCTOBOT
 python scripts/insert_imports.py $env:OCTOBOT_REPOSITORY_DIR/octobot/cli.py
 Copy-Item bin $env:OCTOBOT_REPOSITORY_DIR -recurse
 cd $env:OCTOBOT_REPOSITORY_DIR
-python ../scripts/fetch_nltk_data.py words $NLTK_DATA
+python ../scripts/fetch_nltk_data.py words $env:NLTK_DATA
 python setup.py build_ext --inplace
 python -m PyInstaller bin/start.spec
 Rename-Item dist/OctoBot.exe OctoBot_windows.exe

dev_requirements.txt

@@ -0,0 +1,3 @@
+pytest
+pytest-timeout
+requests

tests/__init__.py

@@ -0,0 +1,54 @@
+#  Drakkar-Software OctoBot-Binary
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+import os
+import platform
+import shutil
+
+
+def is_on_windows():
+    return platform.system() == "Windows"
+
+
+def get_binary_file_path() -> str:
+    if is_on_windows():
+        return "OctoBot_windows_x64.exe\\OctoBot_windows.exe"
+    elif platform.system() == "Darwin":
+        return "./OctoBot_macos_x64/OctoBot_x64"
+    else:
+        return "./OctoBot_linux_x64/OctoBot_x64"
+
+
+def delete_folder_if_exists(folder_path):
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        shutil.rmtree(folder_path)
+
+
+def clear_octobot_previous_folders():
+    try:
+        for folder_path in [
+            "logs",
+            "tentacles",
+            "user"
+        ]:
+            delete_folder_if_exists(folder_path)
+    except PermissionError:
+        # Windows file conflict
+        pass
+
+
+def get_log_file_content(log_file_path="logs/OctoBot.log"):
+    with open(log_file_path, "r") as log_file:
+        return log_file.read()

tests/test_binary.py

@@ -0,0 +1,153 @@
+#  Drakkar-Software OctoBot-Binary
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+import logging
+import os
+import signal
+import subprocess
+from tempfile import TemporaryFile
+
+import pytest
+import requests
+import time
+
+from tests import get_binary_file_path, clear_octobot_previous_folders, get_log_file_content, is_on_windows
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+BINARY_DISABLE_WEB_OPTION = "-nw"
+LOG_CHECKS_MAX_ATTEMPTS = 300
+DEFAULT_TIMEOUT_WINDOW = -95
+
+
+@pytest.fixture
+def start_binary():
+    clear_octobot_previous_folders()
+    with TemporaryFile() as output, TemporaryFile() as err:
+        binary_process = start_binary_process("", output, err)
+        try:
+            yield
+        except Exception:
+            pass
+        finally:
+            terminate_binary(binary_process, output, err)
+
+
+@pytest.fixture
+def start_binary_without_web_app():
+    clear_octobot_previous_folders()
+    with TemporaryFile() as output, TemporaryFile() as err:
+        binary_process = start_binary_process(BINARY_DISABLE_WEB_OPTION, output, err)
+        logger.debug(err.read())
+        try:
+            yield
+        except Exception:
+            pass
+        finally:
+            terminate_binary(binary_process, output, err)
+
+
+def start_binary_process(binary_options, output_file, err_file):
+    logger.debug("Starting binary process...")
+    return subprocess.Popen(f"{get_binary_file_path()}{f' {binary_options}' if binary_options else ''}",
+                            shell=True,
+                            stdout=output_file,
+                            stderr=err_file,
+                            preexec_fn=os.setsid if not is_on_windows() else None)
+
+
+def terminate_binary(binary_process, output_file, err_file):
+    try:
+        logger.info(output_file.read())
+        errors = err_file.read()
+        if errors:
+            logger.error(errors)
+            raise ValueError(f"Error happened during process execution : {errors}")
+    finally:
+        logger.debug("Killing binary process...")
+        if is_on_windows():
+            subprocess.call(["taskkill", "/F", "/IM", "OctoBot_windows.exe"])
+        else:
+            try:
+                os.killpg(os.getpgid(binary_process.pid), signal.SIGTERM)  # Send the signal to all the process groups
+            except ProcessLookupError:
+                binary_process.kill()
+
+
+def multiple_checks(check_method, sleep_time=1, max_attempts=10, **kwargs):
+    attempt = 1
+    while max_attempts >= attempt > 0:
+        try:
+            result = check_method(**kwargs)
+            if result:  # success
+                return
+        except Exception as e:
+            logger.warning(f"Check ({attempt}/{max_attempts}) failed : {e}")
+        finally:
+            attempt += 1
+            try:
+                time.sleep(sleep_time)
+            except KeyboardInterrupt:
+                # Fails when windows is stopping binary
+                pass
+    assert False  # fail
+
+
+def check_endpoint(endpoint_url, expected_code):
+    try:
+        result = requests.get(endpoint_url)
+        return result.status_code == expected_code
+    except requests.exceptions.ConnectionError:
+        logger.warning(f"Failed to get {endpoint_url}")
+        return False
+
+
+def check_logs_content(expected_content: str, should_appear: bool = True):
+    log_content = get_log_file_content()
+    logger.debug(log_content)
+    if should_appear:
+        return expected_content in log_content
+    return expected_content not in log_content
+
+
+@pytest.mark.timeout(100 + DEFAULT_TIMEOUT_WINDOW)
+def test_terms_endpoint(start_binary):
+    multiple_checks(check_endpoint,
+                    max_attempts=100,
+                    endpoint_url="http://localhost:5001/terms",
+                    expected_code=200)
+
+
+@pytest.mark.timeout(LOG_CHECKS_MAX_ATTEMPTS + DEFAULT_TIMEOUT_WINDOW)
+def test_evaluation_state_created(start_binary_without_web_app):
+    multiple_checks(check_logs_content,
+                    max_attempts=LOG_CHECKS_MAX_ATTEMPTS,
+                    expected_content="new state:")
+
+
+@pytest.mark.timeout(LOG_CHECKS_MAX_ATTEMPTS + DEFAULT_TIMEOUT_WINDOW)
+def test_logs_content_has_no_errors(start_binary_without_web_app):
+    multiple_checks(check_logs_content,
+                    max_attempts=LOG_CHECKS_MAX_ATTEMPTS,
+                    expected_content="ERROR",
+                    should_appear=False)
+
+
+@pytest.mark.timeout(LOG_CHECKS_MAX_ATTEMPTS + DEFAULT_TIMEOUT_WINDOW)
+def test_balance_profitability_updated(start_binary_without_web_app):
+    multiple_checks(check_logs_content,
+                    max_attempts=LOG_CHECKS_MAX_ATTEMPTS,
+                    expected_content="BALANCE PROFITABILITY :")