chore: RLN relay tests

.github/workflows/nim_waku_daily.yml

@@ -16,4 +16,5 @@ jobs:
     with:
       node1: "harbor.status.im/wakuorg/nwaku:latest"
       node2: "harbor.status.im/wakuorg/nwaku:latest"
+      additional_nodes: "harbor.status.im/wakuorg/nwaku:latest,harbor.status.im/wakuorg/nwaku:latest,harbor.status.im/wakuorg/nwaku:latest"
       caller: "nim"

.github/workflows/test_common.yml

@@ -49,7 +49,8 @@ jobs:
     - run: pip install -r requirements.txt
 
     - name: Run tests
-      run: pytest -n 4 --reruns 2 --alluredir=allure-results
+      run: |
+        pytest -n 4 --dist loadgroup --reruns 2 --alluredir=allure-results
 
     - name: Get allure history
       if: always()

src/node/api_clients/rest.py

@@ -15,10 +15,19 @@ def rest_call(self, method, endpoint, payload=None):
         headers = {"Content-Type": "application/json"}
         return self.make_request(method, url, headers=headers, data=payload)
 
+    def rest_call_text(self, method, endpoint, payload=None):
+        url = f"http://127.0.0.1:{self._rest_port}/{endpoint}"
+        headers = {"accept": "text/plain"}
+        return self.make_request(method, url, headers=headers, data=payload)
+
     def info(self):
         info_response = self.rest_call("get", "debug/v1/info")
         return info_response.json()
 
+    def health(self):
+        health_response = self.rest_call_text("get", "health")
+        return health_response.text()
+
     def get_peers(self):
         get_peers_response = self.rest_call("get", "admin/v1/peers")
         return get_peers_response.json()

src/node/waku_node.py

@@ -31,7 +31,7 @@ def sanitize_docker_flags(input_flags):
     return output_flags
 
 
-@retry(stop=stop_after_delay(120), wait=wait_fixed(0.5), reraise=True)
+@retry(stop=stop_after_delay(180), wait=wait_fixed(0.5), reraise=True)
 def rln_credential_store_ready(creds_file_path):
     if os.path.exists(creds_file_path):
         return True
@@ -48,7 +48,7 @@ def __init__(self, docker_image, docker_log_prefix=""):
         logger.debug(f"WakuNode instance initialized with log path {self._log_path}")
 
     @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True)
-    def start(self, **kwargs):
+    def start(self, wait_for_node_sec=10, **kwargs):
         logger.debug("Starting Node...")
         self._docker_manager.create_network()
         self._ext_ip = self._docker_manager.generate_random_ext_ip()
@@ -113,6 +113,8 @@ def start(self, **kwargs):
         else:
             logger.info(f"RLN credentials not set or credential store not available, starting without RLN")
 
+        logger.debug(f"Using volumes {self._volumes}")
+
         self._container = self._docker_manager.start_container(
             self._docker_manager.image, self._ports, default_args, self._log_path, self._ext_ip, self._volumes
         )
@@ -121,7 +123,7 @@ def start(self, **kwargs):
         DS.waku_nodes.append(self)
         delay(1)  # if we fire requests to soon after starting the node will sometimes fail to start correctly
         try:
-            self.ensure_ready()
+            self.ensure_ready(timeout_duration=wait_for_node_sec)
         except Exception as ex:
             logger.error(f"REST service did not become ready in time: {ex}")
             raise
@@ -184,10 +186,18 @@ def unpause(self):
             logger.debug(f"Unpause container with id {self._container.short_id}")
             self._container.unpause()
 
-    @retry(stop=stop_after_delay(10), wait=wait_fixed(0.1), reraise=True)
-    def ensure_ready(self):
-        self.info_response = self.info()
-        logger.info("REST service is ready !!")
+    def ensure_ready(self, timeout_duration=10):
+        @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(0.1), reraise=True)
+        def check_ready(node=self):
+            node.info_response = node.info()
+            logger.info("REST service is ready !!")
+
+        check_ready()
+
+    @retry(stop=stop_after_delay(10), wait=wait_fixed(1), reraise=True)
+    def ensure_healthy(self):
+        self.health_response = self.health()
+        logger.info("Node is healthy !!")
 
     def get_enr_uri(self):
         try:
@@ -208,6 +218,9 @@ def get_multiaddr_with_id(self):
     def info(self):
         return self._api.info()
 
+    def health(self):
+        return self._api.health()
+
     def get_peers(self):
         return self._api.get_peers()
 

src/steps/relay.py

@@ -1,20 +1,17 @@
 import inspect
-import os
 from src.libs.custom_logger import get_custom_logger
 from time import time
 import pytest
 import allure
-from src.libs.common import to_base64, delay, gen_step_id
+from src.libs.common import to_base64, delay
 from src.node.waku_message import WakuMessage
 from src.env_vars import (
     NODE_1,
     NODE_2,
     ADDITIONAL_NODES,
     NODEKEY,
-    DEFAULT_NWAKU,
-    RLN_CREDENTIALS,
 )
-from src.node.waku_node import WakuNode, rln_credential_store_ready
+from src.node.waku_node import WakuNode
 from tenacity import retry, stop_after_delay, wait_fixed
 from src.steps.common import StepsCommon
 from src.test_data import VALID_PUBSUB_TOPICS
@@ -45,28 +42,6 @@ def setup_main_relay_nodes(self, request):
         self.add_node_peer(self.node2, [self.multiaddr_with_id])
         self.main_nodes.extend([self.node1, self.node2])
 
-    @pytest.fixture(scope="function")
-    def register_main_rln_relay_nodes(self, request):
-        logger.debug(f"Registering RLN credentials: {inspect.currentframe().f_code.co_name}")
-        self.node1 = WakuNode(DEFAULT_NWAKU, f"node1_{request.cls.test_id}")
-        self.node1.register_rln(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="1")
-        self.node2 = WakuNode(DEFAULT_NWAKU, f"node2_{request.cls.test_id}")
-        self.node2.register_rln(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="2")
-        self.main_nodes.extend([self.node1, self.node2])
-
-    @pytest.fixture(scope="function")
-    def setup_main_rln_relay_nodes(self, request):
-        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
-        self.node1 = WakuNode(DEFAULT_NWAKU, f"node1_{request.cls.test_id}")
-        self.node1.start(relay="true", nodekey=NODEKEY, rln_creds_source=RLN_CREDENTIALS, rln_creds_id="1", rln_relay_membership_index="1")
-        self.enr_uri = self.node1.get_enr_uri()
-        self.node2 = WakuNode(DEFAULT_NWAKU, f"node2_{request.cls.test_id}")
-        self.node2.start(
-            relay="true", discv5_bootstrap_node=self.enr_uri, rln_creds_source=RLN_CREDENTIALS, rln_creds_id="2", rln_relay_membership_index="1"
-        )
-        self.add_node_peer(self.node2, [self.multiaddr_with_id])
-        self.main_nodes.extend([self.node1, self.node2])
-
     @pytest.fixture(scope="function")
     def setup_optional_relay_nodes(self, request):
         logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
@@ -156,19 +131,3 @@ def delete_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list):
     def subscribe_and_publish_with_retry(self, node_list, pubsub_topic_list):
         self.ensure_relay_subscriptions_on_nodes(node_list, pubsub_topic_list)
         self.check_published_message_reaches_relay_peer()
-
-    @allure.step
-    def register_rln_single_node(self, **kwargs):
-        logger.debug("Registering RLN credentials for single node")
-        self.node1 = WakuNode(DEFAULT_NWAKU, f"node1_{gen_step_id()}")
-        self.node1.register_rln(rln_creds_source=kwargs["rln_creds_source"], rln_creds_id=kwargs["rln_creds_id"])
-
-    @allure.step
-    def check_rln_registration(self, key_id):
-        current_working_directory = os.getcwd()
-        creds_file_path = f"{current_working_directory}/keystore_{key_id}/keystore.json"
-        try:
-            rln_credential_store_ready(creds_file_path)
-        except Exception as ex:
-            logger.error(f"Credentials at {creds_file_path} not available: {ex}")
-            raise

src/steps/rln.py

@@ -0,0 +1,178 @@
+import os
+import inspect
+import pytest
+import allure
+
+from src.node.waku_message import WakuMessage
+from src.steps.common import StepsCommon
+from src.test_data import PUBSUB_TOPICS_RLN
+from src.env_vars import DEFAULT_NWAKU, RLN_CREDENTIALS, NODEKEY, NODE_1, NODE_2, ADDITIONAL_NODES
+from src.libs.common import gen_step_id, delay
+from src.libs.custom_logger import get_custom_logger
+from src.node.waku_node import WakuNode, rln_credential_store_ready
+
+logger = get_custom_logger(__name__)
+
+
+class StepsRLN(StepsCommon):
+    test_pubsub_topic = PUBSUB_TOPICS_RLN[0]
+    test_content_topic = "/test/1/waku-rln-relay/proto"
+    test_payload = "RLN relay works!!"
+
+    main_nodes = []
+    optional_nodes = []
+    multiaddr_list = []
+    lightpush_nodes = []
+
+    @pytest.fixture(scope="function")
+    def register_main_rln_relay_nodes(self, request):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.register_rln_single_node(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="1")
+        self.register_rln_single_node(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="2")
+
+    @pytest.fixture(scope="function")
+    def register_optional_rln_relay_nodes(self, request):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.register_rln_single_node(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="3")
+        self.register_rln_single_node(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="4")
+        self.register_rln_single_node(rln_creds_source=RLN_CREDENTIALS, rln_creds_id="5")
+
+    @allure.step
+    def setup_main_rln_relay_nodes(self, **kwargs):
+        self.setup_first_rln_relay_node(**kwargs)
+        self.setup_second_rln_relay_node(**kwargs)
+
+    @allure.step
+    def setup_first_rln_relay_node(self, **kwargs):
+        self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}")
+        self.node1.start(
+            relay="true",
+            nodekey=NODEKEY,
+            rln_creds_source=RLN_CREDENTIALS,
+            rln_creds_id="1",
+            rln_relay_membership_index="1",
+            **kwargs,
+        )
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        self.main_nodes.extend([self.node1])
+
+        self.multiaddr_list.extend([self.node1.get_multiaddr_with_id()])
+
+    @allure.step
+    def setup_second_rln_relay_node(self, **kwargs):
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(
+            relay="true",
+            discv5_bootstrap_node=self.enr_uri,
+            rln_creds_source=RLN_CREDENTIALS,
+            rln_creds_id="2",
+            rln_relay_membership_index="1",
+            **kwargs,
+        )
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node2])
+
+    @allure.step
+    def setup_optional_rln_relay_nodes(self, **kwargs):
+        if ADDITIONAL_NODES:
+            nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")]
+        else:
+            pytest.skip("ADDITIONAL_NODES is empty, cannot run test")
+        if len(nodes) > 3:
+            logger.debug("More than 3 nodes are not supported for RLN tests, using first 3")
+            nodes = nodes[:3]
+        for index, node in enumerate(nodes):
+            node = WakuNode(node, f"node{index + 3}_{self.test_id}")
+            node.start(
+                relay="true",
+                discv5_bootstrap_node=self.enr_uri,
+                rln_creds_source=RLN_CREDENTIALS,
+                rln_creds_id=f"{index + 3}",
+                rln_relay_membership_index="1",
+                **kwargs,
+            )
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)
+
+    @allure.step
+    def setup_second_lightpush_node(self, relay="false", **kwargs):
+        self.light_push_node2 = WakuNode(NODE_2, f"lightpush_node2_{self.test_id}")
+        self.light_push_node2.start(relay=relay, discv5_bootstrap_node=self.enr_uri, lightpush="true", lightpushnode=self.multiaddr_list[0], **kwargs)
+        if relay == "true":
+            self.main_nodes.extend([self.light_push_node2])
+        self.lightpush_nodes.extend([self.light_push_node2])
+        self.add_node_peer(self.light_push_node2, self.multiaddr_list)
+
+    @allure.step
+    def setup_first_relay_node(self, **kwargs):
+        self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}")
+        self.node1.start(relay="true", nodekey=NODEKEY, **kwargs)
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        self.main_nodes.extend([self.node1])
+
+    @allure.step
+    def setup_second_relay_node(self, **kwargs):
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(
+            relay="true",
+            discv5_bootstrap_node=self.enr_uri,
+            **kwargs,
+        )
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node2])
+
+    @allure.step
+    def register_rln_single_node(self, **kwargs):
+        logger.debug("Registering RLN credentials for single node")
+        self.node1 = WakuNode(DEFAULT_NWAKU, f"node1_{gen_step_id()}")
+        self.node1.register_rln(rln_creds_source=kwargs["rln_creds_source"], rln_creds_id=kwargs["rln_creds_id"])
+
+    @allure.step
+    def check_rln_registration(self, key_id):
+        current_working_directory = os.getcwd()
+        creds_file_path = f"{current_working_directory}/keystore_{key_id}/keystore.json"
+        try:
+            rln_credential_store_ready(creds_file_path)
+        except Exception as ex:
+            logger.error(f"Credentials at {creds_file_path} not available: {ex}")
+            raise
+
+    @allure.step
+    def publish_message(self, message=None, pubsub_topic=None, sender=None, use_lightpush=False):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if not sender:
+            sender = self.node1
+
+        if use_lightpush:
+            payload = self.create_payload(pubsub_topic, message)
+            sender.send_light_push_message(payload)
+        else:
+            sender.send_relay_message(message, pubsub_topic)
+
+    @allure.step
+    def ensure_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list):
+        for node in node_list:
+            node.set_relay_subscriptions(pubsub_topic_list)
+
+    @allure.step
+    def subscribe_main_relay_nodes(self):
+        self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic])
+
+    @allure.step
+    def subscribe_optional_relay_nodes(self):
+        self.ensure_relay_subscriptions_on_nodes(self.optional_nodes, [self.test_pubsub_topic])
+
+    @allure.step
+    def create_payload(self, pubsub_topic=None, message=None, **kwargs):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        payload = {"pubsubTopic": pubsub_topic, "message": message}
+        payload.update(kwargs)
+        return payload

src/test_data.py

@@ -148,3 +148,5 @@
     {"description": "ISO 8601 timestamp", "value": "2023-12-26T10:58:51", "valid_for": []},
     {"description": "Missing", "value": None, "valid_for": ["gowaku"]},
 ]
+
+PUBSUB_TOPICS_RLN = ["/waku/2/rs/1/0"]