From 930028e8b7028fe07c1964cdb2416ec3fad2f423 Mon Sep 17 00:00:00 2001 From: Evan Adam Date: Fri, 9 Aug 2024 10:38:15 +0200 Subject: [PATCH 1/6] tests(Gorgone): test action module to retrieve result of command Refs:MON-139728 --- gorgone/tests/robot/resources/LogResearch.py | 60 ++++++++- gorgone/tests/robot/tests/core/action.robot | 121 +++++++++++++++++++ 2 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 gorgone/tests/robot/tests/core/action.robot diff --git a/gorgone/tests/robot/resources/LogResearch.py b/gorgone/tests/robot/resources/LogResearch.py index e6e6ccd54d0..899ff28805d 100644 --- a/gorgone/tests/robot/resources/LogResearch.py +++ b/gorgone/tests/robot/resources/LogResearch.py @@ -1,13 +1,69 @@ +# +# Copyright 2024 Centreon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For more information : contact@centreon.com +# + from robot.api import logger import re import time from dateutil import parser from datetime import datetime, timedelta -from robot.libraries.BuiltIn import BuiltIn +import requests + + +def ctn_get_api_log_with_timeout(token: str, node_path='', host='http://127.0.0.1:8085', timeout=15): + """! Query gorgone log api until the response contain a log with code 2 (success) or 1 (failure) + @param token: token to search in the api + @param node_path: part of the api url defining if we use the local gorgone or another one, ex node/2/ + @param timeout: time out in second + @param host: gorgone api url with the port + @return True(when output of the command is found)/False(on failure or timeout), + and a json object containing the incriminated log for failure or success. + """ + limit_date = time.time() + timeout + api_json = [] + while time.time() < limit_date: + time.sleep(1) + uri = host + "/api/" + node_path + "log/" + token + response = requests.get(uri) + (status, output) = parse_json_response(response) + if status == '': + continue + return status, output + + return False, api_json["data"] + + +def parse_json_response(response): + api_json = response.json() + # http code should either be 200 for success or 404 for no log found if we are too early. + # as the time of writing, status code is always 200 because webapp autodiscovery module always expect a 200. + if response.status_code != 200 and response.status_code != 404: + return False, api_json -TIMEOUT = 30 + if 'error' in api_json and api_json['error'] == "no_log": + return '', '' + for log_detail in api_json["data"]: + if log_detail["code"] == 2: + return False, log_detail + if log_detail["code"] == 100: + return True, log_detail +# these function search log in the gorgone log file def ctn_find_in_log_with_timeout(log: str, content, timeout=20, date=-1, regex=False): """! search a pattern in log from date param @param log: path of the log file diff --git a/gorgone/tests/robot/tests/core/action.robot b/gorgone/tests/robot/tests/core/action.robot new file mode 100644 index 00000000000..6059465157f --- /dev/null +++ b/gorgone/tests/robot/tests/core/action.robot @@ -0,0 +1,121 @@ +# +# Copyright 2024 Centreon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For more information : contact@centreon.com +# + +*** Settings *** +Documentation test gorgone action module on local and distant target +Resource ${CURDIR}${/}..${/}..${/}resources${/}import.resource +Test Timeout 220s + +# @TODO : I know it's possible to have a remote server managing some poller. For now we don't test this case, but it should be tested and documented. +*** Test Cases *** +action module with ${communication_mode} communcation mode + [Documentation] test action on distant node, no whitelist configured + @{process_list} Create List ${communication_mode}_gorgone_central ${communication_mode}_gorgone_poller_2 + [Teardown] Stop Gorgone And Remove Gorgone Config @{process_list} sql_file=${ROOT_CONFIG}db_delete_poller.sql + + @{central_config} Create List ${ROOT_CONFIG}actions.yaml + @{poller_config} Create List ${ROOT_CONFIG}actions.yaml + Setup Two Gorgone Instances + ... central_config=${central_config} + ... communication_mode=${communication_mode} + ... central_name=${communication_mode}_gorgone_central + ... poller_name=${communication_mode}_gorgone_poller_2 + ... poller_config=${poller_config} + + # first we test the api without waiting for the output of the command. + # check by default the api launch the query in local + Test Async Action Module + # check the central can execute a command and send back the output + Test Async Action Module node_path=nodes/1/ + # check a distant poller can execute a command and send back the output + ${start_date} Get Current Date increment=-10s + Test Async Action Module node_path=nodes/2/ + # we need to check it is the poller and not the central that have done the action. + ${log_poller2_query} Create List Robot test write with param: for node nodes/2/ + ${logs_poller} Ctn Find In Log With Timeout log=/var/log/centreon-gorgone/${communication_mode}_gorgone_poller_2/gorgoned.log content=${log_poller2_query} date=${start_date} timeout=10 + Should Be True ${logs_poller} Didn't found the logs in the poller file : ${logs_poller} + + # Now we test the action api by waiting for the command output in one call. + # This make gorgone wait for 3 seconds before querying for logs, and wait again 0.5 seconds for log to be received by central. + # On my machine the sync_wait was at least 0.22 seconds to work sometime, it always worked with 0.5s. + # In real world where poller is not on the same server the delay will be greater and more random, + # so the async method should be privileged. + ${get_params}= Set Variable ?log_wait=3000000&sync_wait=500000 + Test Sync Action Module communication_mode=${communication_mode} get_params=${get_params} + Test Sync Action Module communication_mode=${communication_mode} get_params=${get_params} node_path=nodes/1/ + Test Sync Action Module communication_mode=${communication_mode} get_params=${get_params} node_path=nodes/2/ + # we need to check it is the poller and not the central that have done the action. + ${start_date} Get Current Date increment=-10s + ${log_poller2_query_sync} Create List Robot test write with param:${get_params} for node nodes/2/ + ${logs_poller} Ctn Find In Log With Timeout log=/var/log/centreon-gorgone/${communication_mode}_gorgone_poller_2/gorgoned.log content=${log_poller2_query_sync} date=${start_date} timeout=10 + Should Be True ${logs_poller} Didn't found the logs in the poller file: ${logs_poller} + + Run rm /tmp/actionLogs + + Examples: communication_mode -- + ... push_zmq + ... pullwss + ... pull + +*** Keywords *** +Test Sync Action Module + [Arguments] ${communication_mode}= ${get_params}= ${node_path}= + + ${action_api_result}= Post Action Endpoint node_path=${node_path} get_params=${get_params} + ${status} ${logs} Parse Json Response ${action_api_result} + Check Action Api Do Something ${status} ${logs} ${node_path} ${get_params} + + +Test Async Action Module + [Documentation] This make an api call to write to a dummy file and output a string. as gorgone central and poller and robot are executed on the same host we can access the file to check the result. + [Arguments] ${node_path}=${EMPTY} + ${action_api_result}= Post Action Endpoint node_path=${node_path} + # need to get the data from the token with getlog. + # this call multiples time the api until the response is available. + ${status} ${logs} Ctn Get Api Log With Timeout token=${action_api_result.json()}[token] node_path=${node_path} + Check Action Api Do Something ${status} ${logs} ${node_path} ${EMPTY} + + +Post Action Endpoint + [Arguments] ${node_path}=${EMPTY} ${get_params}=${EMPTY} + + # Ideally, Gorgone should not allow any bash interpretation on command it execute. + # As there is a whitelist in gorgone, if there was no bash interpretation we could allow only our required binary and be safe. + # As gorgone always had bash interpretation available, most of the internal use of this module use redirection, pipe or other sh feature. + ${bodycmd}= Create Dictionary command=echo 'Robot test write with param:${get_params} for node ${node_path}' | tee -a /tmp/actionLogs + ${body}= Create List ${bodycmd} + ${result} POST http://127.0.0.1:8085/api/${node_path}core/action/command${get_params} json=${body} + RETURN ${result} + + +Check Action Api Do Something + [Arguments] ${status} ${logs} ${node_path} ${get_params} + + Should Be True ${status} No log found in the gorgone api or the command failed. + # the log api send back a json containing a list of log, with for each logs the token, id, creation time (ctime), status code(code), and data (among other thing) + # data is a stringified json that need to be evaluated separately. + ${internal_json}= Evaluate json.loads("""${logs}[data]""") json + + Should Be Equal As Numbers 0 ${internal_json}[result][exit_code] + Should Be Equal As Strings + ... Robot test write with param:${get_params} for node ${node_path} + ... ${internal_json}[result][stdout] + ... output of the gorgone action api should be the bash command output. + + ${file_nb_line}= Run grep 'Robot test write with param:${get_params} for node ${node_path}\$' /tmp/actionLogs | wc -l + Should Be Equal 1 ${file_nb_line} command launched with gorgone api should set only one line in the file per tests From b033b9b136086383b3c47cea446a191850114c65 Mon Sep 17 00:00:00 2001 From: Evan Adam Date: Wed, 14 Aug 2024 12:42:08 +0200 Subject: [PATCH 2/6] doc(Gorgone): Fix documentation of action module Refs:MON-139728 --- gorgone/docs/api.md | 64 +++++++++++++++-------------- gorgone/docs/modules/core/action.md | 18 +++++--- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/gorgone/docs/api.md b/gorgone/docs/api.md index 4ee643f273d..d98a8ea86a8 100644 --- a/gorgone/docs/api.md +++ b/gorgone/docs/api.md @@ -6,14 +6,14 @@ Centreon Gorgone provides a RestAPI through its HTTP server module. ### Get Nodes Connection Status -| Endpoint | Method | -| :- | :- | -| /internal/constatus | `GET` | +| Endpoint | Method | +|:--------------------|:-------| +| /internal/constatus | `GET` | #### Headers -| Header | Value | -| :- | :- | +| Header | Value | +|:-------|:-----------------| | Accept | application/json | #### Example @@ -42,14 +42,14 @@ curl --request GET "https://hostname:8443/api/internal/constatus" \ ### Get Public Key Thumbprint -| Endpoint | Method | -| :- | :- | -| /internal/thumbprint | `GET` | +| Endpoint | Method | +|:---------------------|:-------| +| /internal/thumbprint | `GET` | #### Headers -| Header | Value | -| :- | :- | +| Header | Value | +|:-------|:-----------------| | Accept | application/json | #### Example @@ -73,14 +73,14 @@ curl --request GET "https://hostname:8443/api/internal/thumbprint" \ ### Get Runtime Informations And Statistics -| Endpoint | Method | -| :- | :- | -| /internal/information | `GET` | +| Endpoint | Method | +|:----------------------|:-------| +| /internal/information | `GET` | #### Headers -| Header | Value | -| :- | :- | +| Header | Value | +|:-------|:-----------------| | Accept | application/json | #### Example @@ -158,6 +158,7 @@ The available endpoints depend on which modules are loaded. Endpoints are basically built from: * API root, +* optional target node, local if not present ( `/nodes/:nodeid/` ) * Module's namespace, * Module's name, * Action @@ -175,7 +176,7 @@ curl --request POST "https://hostname:8443/api/core/action/command" \ ]" ``` -Find more informations directly from modules documentations [here](../docs/modules.md). +Find more informations directly from modules documentations [here](./modules.md). As Centreon Gorgone is asynchronous, those endpoints will return a token corresponding to the action. @@ -187,7 +188,7 @@ As Centreon Gorgone is asynchronous, those endpoints will return a token corresp } ``` -That being said, its possible to make Gorgone work synchronously by providing two parameters. +That being said, it is possible to make Gorgone work synchronously by providing two parameters. First one is `log_wait` with a numeric value in microseconds: this value defines the amount of time the API will wait before trying to retrieve log results. @@ -202,7 +203,7 @@ Note: the `sync_wait` parameter is induced if you ask for a log directly specify Using the `/core/action/command` endpoint with `log_wait` parameter set to 100000: ```bash -curl --request POST "https://hostname:8443/api/core/action/command&log_wait=100000" \ +curl --request POST "https://hostname:8443/api/core/action/command?log_wait=100000" \ --header "Accept: application/json" \ --header "Content-Type: application/json" \ --data "[ @@ -219,12 +220,12 @@ This call will ask for the API to execute an action and will give a result after Note: there is no need for logs synchronisation when dealing with local actions. -##### Launch a command remotly and wait for the result +##### Launch a command remotely and wait for the result Using the `/nodes/:id/core/action/command` endpoint with `log_wait` parameter set to 100000: ```bash -curl --request POST "https://hostname:8443/api/nodes/2/core/action/command&log_wait=100000&sync_wait=200000" \ +curl --request POST "https://hostname:8443/api/nodes/2/core/action/command?log_wait=100000&sync_wait=200000" \ --header "Accept: application/json" \ --header "Content-Type: application/json" \ --data "[ @@ -241,23 +242,24 @@ This call will ask for the API to execute an action on the node with ID 2, will ## Log endpoint -To retrieve the logs, a specific endpoint can be called as follow. +To retrieve the logs, a specific endpoint can be called as follows. -| Endpoint | Method | -| :- | :- | -| /log/:token | `GET` | +| Endpoint | Method | +|:------------------------------|:-------| +| /api/nodes/:nodeid/log/:token | `GET` | #### Headers -| Header | Value | -| :- | :- | -| Accept | application/json | +| Header | Value | +|:-------|:------------------| +| Accept | application/json | #### Path variables -| Variable | Description | -| :- | :- | -| token | Token of the action | +| Variable | Description | +|:---------|:---------------------------| +| token | Token of the action | +| nodeid | node id to search log into | #### Examples @@ -271,7 +273,7 @@ curl --request GET "https://hostname:8443/api/nodes/2/log/3f25bc3a797fe989d1fb05 --header "Accept: application/json" ``` -This second example will force logs synchonisation before looking for results to retrieve. Default temporisation is 10ms and can be changed by providing `sync_wait` parameter. +This second example will force logs synchronisation before looking for results to retrieve. Default wait time is 10ms and can be changed by providing `sync_wait` parameter. #### Response example diff --git a/gorgone/docs/modules/core/action.md b/gorgone/docs/modules/core/action.md index c885dcd0cac..db21bbe971c 100644 --- a/gorgone/docs/modules/core/action.md +++ b/gorgone/docs/modules/core/action.md @@ -7,7 +7,7 @@ This module aims to execute actions on the server running the Gorgone daemon or ## Configuration | Directive | Description | Default value | -| :--------------- | :------------------------------------------------------------- | :------------ | +|:-----------------|:---------------------------------------------------------------|:--------------| | command_timeout | Time in seconds before a command is considered timed out | `30` | | whitelist_cmds | Boolean to enable commands whitelist | `false` | | allowed_cmds | Regexp list of allowed commands | | @@ -38,7 +38,7 @@ allowed_cmds: ## Events | Event | Description | -| :---------- | :-------------------------------------------------------------------------------------- | +|:------------|:----------------------------------------------------------------------------------------| | ACTIONREADY | Internal event to notify the core | | PROCESSCOPY | Process file or archive received from another daemon | | COMMAND | Execute a shell command on the server running the daemon or on another server using SSH | @@ -48,20 +48,20 @@ allowed_cmds: ### Execute a command line | Endpoint | Method | -| :------------------- | :----- | +|:---------------------|:-------| | /core/action/command | `POST` | #### Headers | Header | Value | -| :----------- | :--------------- | +|:-------------|:-----------------| | Accept | application/json | | Content-Type | application/json | #### Body | Key | Value | -| :---------------- | :------------------------------------------------------- | +|:------------------|:---------------------------------------------------------| | command | Command to execute | | timeout | Time in seconds before a command is considered timed out | | continue_on_error | Behaviour in case of execution issue | @@ -76,8 +76,11 @@ allowed_cmds: ] ``` + #### Example +See a complete exemple of this endpoint in the [api documentation](../../api.md) + ```bash curl --request POST "https://hostname:8443/api/core/action/command" \ --header "Accept: application/json" \ @@ -88,3 +91,8 @@ curl --request POST "https://hostname:8443/api/core/action/command" \ } ]" ``` +Output : +```bash +{"token":"b3f825f87d64764316d872c59e4bae69299b0003f6e5d27bbc7de4e27c50eb65fc17440baf218578343eff7f4d67f7e98ab6da40b050a2635bb735c7cec276bd"} +``` + From 5f373f7e84b947d3bf21e18ec25df5787ea55458 Mon Sep 17 00:00:00 2001 From: Evan Adam Date: Wed, 14 Aug 2024 09:22:21 +0200 Subject: [PATCH 3/6] ci(Gorgone): remove os specific db name as database are not shared across jobs Refs:MON-139728 --- .github/workflows/gorgone.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gorgone.yml b/.github/workflows/gorgone.yml index a60905c0ee8..6c090e1fd5f 100644 --- a/.github/workflows/gorgone.yml +++ b/.github/workflows/gorgone.yml @@ -222,15 +222,15 @@ jobs: - name: Create databases run: | - mysql -h mariadb -u root -ppassword -e "CREATE DATABASE \`centreon${{ matrix.distrib }}\`" - mysql -h mariadb -u root -ppassword -e "CREATE DATABASE \`centreon-storage${{ matrix.distrib }}\`" - mysql -h mariadb -u root -ppassword -e "GRANT ALL PRIVILEGES ON centreon${{ matrix.distrib }}.* TO 'centreon'@'%'" - mysql -h mariadb -u root -ppassword -e "GRANT ALL PRIVILEGES ON \`centreon-storage${{ matrix.distrib }}\`.* TO 'centreon'@'%'" - mysql -h mariadb -u root -ppassword 'centreon${{ matrix.distrib }}' < centreon/centreon/www/install/createTables.sql - mysql -h mariadb -u root -ppassword 'centreon-storage${{ matrix.distrib }}' < centreon/centreon/www/install/createTablesCentstorage.sql + mysql -h mariadb -u root -ppassword -e "CREATE DATABASE \`centreon\`" + mysql -h mariadb -u root -ppassword -e "CREATE DATABASE \`centreon-storage\`" + mysql -h mariadb -u root -ppassword -e "GRANT ALL PRIVILEGES ON centreon.* TO 'centreon'@'%'" + mysql -h mariadb -u root -ppassword -e "GRANT ALL PRIVILEGES ON \`centreon-storage\`.* TO 'centreon'@'%'" + mysql -h mariadb -u root -ppassword 'centreon' < centreon/centreon/www/install/createTables.sql + mysql -h mariadb -u root -ppassword 'centreon-storage' < centreon/centreon/www/install/createTablesCentstorage.sql - name: Run tests - run: robot -v 'DBHOST:mariadb' -v 'DBNAME:centreon${{ matrix.distrib }}' -v 'DBNAME_STORAGE:centreon-storage${{ matrix.distrib }}' -v 'DBUSER:centreon' gorgone/tests + run: robot -v 'DBHOST:mariadb' -v 'DBNAME:centreon' -v 'DBNAME_STORAGE:centreon-storage' -v 'DBUSER:centreon' gorgone/tests - name: Upload gorgone and robot debug artifacts if: failure() From 8500fbf4f2d96f7dbd7179dd9dade34b7b0040da Mon Sep 17 00:00:00 2001 From: Evan Adam Date: Wed, 14 Aug 2024 14:31:08 +0200 Subject: [PATCH 4/6] test(Gorgone): fix pull test to start poller first, so there is no failed ping from the host Refs:MON-139728 --- gorgone/tests/robot/resources/resources.resource | 1 + 1 file changed, 1 insertion(+) diff --git a/gorgone/tests/robot/resources/resources.resource b/gorgone/tests/robot/resources/resources.resource index 23d01e07a9c..89d58688292 100644 --- a/gorgone/tests/robot/resources/resources.resource +++ b/gorgone/tests/robot/resources/resources.resource @@ -169,6 +169,7 @@ Setup Two Gorgone Instances Start Gorgone debug ${poller_name} Start Gorgone debug ${central_name} Wait Until Port Is Bind 5556 + Check Poller Is Connected port=5556 expected_nb=2 Check Poller Communicate 2 END From 2bf9d7fcf50ab1ef9a100f5cb2cc7427a2affae1 Mon Sep 17 00:00:00 2001 From: Evan-Adam <152897682+Evan-Adam@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:26:29 +0200 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: cg-tw <83637804+cg-tw@users.noreply.github.com> --- gorgone/tests/robot/resources/LogResearch.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gorgone/tests/robot/resources/LogResearch.py b/gorgone/tests/robot/resources/LogResearch.py index 899ff28805d..0fe4afbd9d4 100644 --- a/gorgone/tests/robot/resources/LogResearch.py +++ b/gorgone/tests/robot/resources/LogResearch.py @@ -25,11 +25,11 @@ def ctn_get_api_log_with_timeout(token: str, node_path='', host='http://127.0.0.1:8085', timeout=15): - """! Query gorgone log api until the response contain a log with code 2 (success) or 1 (failure) - @param token: token to search in the api - @param node_path: part of the api url defining if we use the local gorgone or another one, ex node/2/ - @param timeout: time out in second - @param host: gorgone api url with the port + """! Query gorgone log API until the response contains a log with code 2 (success) or 1 (failure) + @param token: token to search in the API + @param node_path: part of the API URL defining if we use the local gorgone or another one, ex node/2/ + @param timeout: timeout in seconds + @param host: gorgone API URL with the port @return True(when output of the command is found)/False(on failure or timeout), and a json object containing the incriminated log for failure or success. """ From 060fc05428d169093536f9c5a958399c4a3765ae Mon Sep 17 00:00:00 2001 From: Evan Adam Date: Wed, 21 Aug 2024 16:52:52 +0200 Subject: [PATCH 6/6] fix(Gorgone): change after review Refs:MON-139728 --- gorgone/tests/robot/tests/core/action.robot | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gorgone/tests/robot/tests/core/action.robot b/gorgone/tests/robot/tests/core/action.robot index 6059465157f..49a155420b1 100644 --- a/gorgone/tests/robot/tests/core/action.robot +++ b/gorgone/tests/robot/tests/core/action.robot @@ -56,9 +56,9 @@ action module with ${communication_mode} communcation mode # In real world where poller is not on the same server the delay will be greater and more random, # so the async method should be privileged. ${get_params}= Set Variable ?log_wait=3000000&sync_wait=500000 - Test Sync Action Module communication_mode=${communication_mode} get_params=${get_params} - Test Sync Action Module communication_mode=${communication_mode} get_params=${get_params} node_path=nodes/1/ - Test Sync Action Module communication_mode=${communication_mode} get_params=${get_params} node_path=nodes/2/ + Test Sync Action Module get_params=${get_params} + Test Sync Action Module get_params=${get_params} node_path=nodes/1/ + Test Sync Action Module get_params=${get_params} node_path=nodes/2/ # we need to check it is the poller and not the central that have done the action. ${start_date} Get Current Date increment=-10s ${log_poller2_query_sync} Create List Robot test write with param:${get_params} for node nodes/2/ @@ -74,7 +74,7 @@ action module with ${communication_mode} communcation mode *** Keywords *** Test Sync Action Module - [Arguments] ${communication_mode}= ${get_params}= ${node_path}= + [Arguments] ${get_params}= ${node_path}= ${action_api_result}= Post Action Endpoint node_path=${node_path} get_params=${get_params} ${status} ${logs} Parse Json Response ${action_api_result}