Skip to content
CI

PBM-397: warn about parallel agent for the same node #486

Sign in to view logs

PBM-397: warn about parallel agent for the same node

PBM-397: warn about parallel agent for the same node #486

GitHub Actions / JUnit Test Report failed Sep 23, 2024 in 0s

15 tests run, 3 passed, 1 skipped, 11 failed.

Annotations

Check failure on line 100 in psmdb-testing/pbm-functional/pytest/test_PBM-1155.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_PBM-1155.test_incremental_PBM_T245 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fdd0c950>

    @pytest.mark.timeout(6000, func_only=True)
    def test_incremental_PBM_T245(start_cluster,cluster):
        cluster.check_pbm_status()
        time.sleep(10)
        cluster.make_backup("incremental --base")
        for i in range(20):
            time.sleep(1)
            pymongo.MongoClient(cluster.connection)['test']['test'].insert_one({'data': random.randbytes(1024)})
            backup=cluster.make_backup("incremental")
    
>       cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_PBM-1155.py:100: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fdd0c950>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 154 in psmdb-testing/pbm-functional/pytest/test_PBM-979.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_PBM-979.test_incremental_PBM_T234 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fe0cdcd0>

    @pytest.mark.timeout(300,func_only=True)
    def test_incremental_PBM_T234(start_cluster,cluster):
        time.sleep(5)
        cluster.check_pbm_status()
        init_backup=cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
        logs=cluster.exec_pbm_cli("logs -t 200 -n rs1/rs103:27017 -e backup -o json").stdout
        Cluster.log("Logs from hidden node:\n" + logs)
        assert init_backup in logs
        assert backup in logs
        cluster.make_restore(backup,restart_cluster=True)
        time.sleep(5)
>       cluster.check_pbm_status()

test_PBM-979.py:154: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fe0cdcd0>

    def check_pbm_status(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm status --out=json')
        parsed_result = json.loads(result)
        Cluster.log("PBM status: \n" + str(parsed_result['cluster']))
        #Cluster.log(json.dumps(parsed_result['cluster'], indent=4))
        hosts = []
        for replicaset in parsed_result['cluster']:
            for host in replicaset['nodes']:
                if host['role'] != "A":
                    hosts.append(host)
>                   assert host['ok'] == True
E                   AssertionError

cluster.py:728: AssertionError

Check failure on line 125 in psmdb-testing/pbm-functional/pytest/test_azurite.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_azurite.test_incremental 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fdef79d0>

    @pytest.mark.timeout(300, func_only=True)
    def test_incremental(start_cluster, cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup = cluster.make_backup("incremental")
        result = pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
>       cluster.make_restore(backup, restart_cluster=True,check_pbm_status=True)

test_azurite.py:125: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fdef79d0>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 57 in psmdb-testing/pbm-functional/pytest/test_encryption.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_encryption.test_incremental 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fdcfd550>

    @pytest.mark.timeout(300,func_only=True)
    def test_incremental(start_cluster,cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
>       cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_encryption.py:57: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fdcfd550>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 103 in psmdb-testing/pbm-functional/pytest/test_fresh_sharded.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_fresh_sharded.test_incremental_PBM_T209 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31ff48a110>
newcluster = <cluster.Cluster object at 0x7f31ff48bfd0>

    @pytest.mark.timeout(600,func_only=True)
    def test_incremental_PBM_T209(start_cluster,cluster,newcluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("incremental")
        cluster.destroy()
    
        newcluster.make_resync()
>       newcluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_fresh_sharded.py:103: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31ff48bfd0>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 57 in psmdb-testing/pbm-functional/pytest/test_kmip.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_kmip.test_incremental_PBM_T201 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fe09dd50>

    @pytest.mark.timeout(300,func_only=True)
    def test_incremental_PBM_T201(start_cluster,cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
>       cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_kmip.py:57: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fe09dd50>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 103 in psmdb-testing/pbm-functional/pytest/test_remap_sharded.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_remap_sharded.test_incremental_PBM_T243 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fde084d0>
newcluster = <cluster.Cluster object at 0x7f31fde0a590>

    @pytest.mark.timeout(600,func_only=True)
    def test_incremental_PBM_T243(start_cluster,cluster,newcluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("incremental")
        backup = backup + ' --replset-remapping="newrs1=rs1,newrs2=rs2,newrscfg=rscfg"'
        cluster.destroy()
    
        newcluster.make_resync()
>       newcluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_remap_sharded.py:103: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fde0a590>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 148 in psmdb-testing/pbm-functional/pytest/test_sharded.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_sharded.test_incremental 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fdda3950>

    @pytest.mark.timeout(300,func_only=True)
    def test_incremental(start_cluster,cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup = cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
>       cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_sharded.py:148: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fdda3950>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 105 in psmdb-testing/pbm-functional/pytest/test_timeseries.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_timeseries.test_incremental_PBM_T262 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fd7c6a90>

    @pytest.mark.timeout(600,func_only=True)
    def test_incremental_PBM_T262(start_cluster,cluster):
        cluster.check_pbm_status()
        assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test1").get("sharded", False)
        assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test2").get("sharded", False)
        for i in range(10):
            pymongo.MongoClient(cluster.connection)["test"]["test1"].insert_one({"timestamp": datetime.now(), "data": i})
            time.sleep(0.1)
        base_backup=cluster.make_backup("incremental --base")
        for i in range(10):
            pymongo.MongoClient(cluster.connection)["test"]["test2"].insert_one({"timestamp": datetime.now(), "data": i})
            time.sleep(0.1)
        backup=cluster.make_backup("incremental")
        pymongo.MongoClient(cluster.connection).drop_database('test')
>       cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True)

test_timeseries.py:105: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fd7c6a90>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError

Check failure on line 97 in psmdb-testing/pbm-functional/pytest/test_upgrade.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_upgrade.test_incremental 

AssertionError: Starting restore 2024-09-23T19:22:44.521484112Z from '2024-09-23T19:22:14Z'........................................................................................................................Error: no confirmation that restore has successfully started. Replsets status:
  <no replset has started restore>
  
  
assert False
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fdbef450>

    @pytest.mark.timeout(600,func_only=True)
    def test_incremental(start_cluster,cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup = cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
        cluster.upgrade()
        cluster.check_pbm_status()
        try:
>           cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_upgrade.py:88: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fdbef450>, name = '2024-09-23T19:22:14Z'
kwargs = {'check_pbm_status': True, 'restart_cluster': True}
client = MongoClient(host=['mongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7f31fe8451d0>, exit_status=1, command=b'time...'Error: no confirmation that restore has successfully started. Replsets status:\n<no replset has started restore>\n\n')
n = <testinfra.host.Host docker://rscfg01>, timeout = 240, error = ''
host = 'rscfg03', container = <Container: 5b669bb2af57>

    def make_restore(self, name, **kwargs):
        if self.layout == "sharded":
            client = pymongo.MongoClient(self.connection)
            result = client.admin.command("balancerStop")
            client.close()
            Cluster.log("Stopping balancer: " + str(result))
            self.stop_mongos()
        self.stop_arbiters()
        n = testinfra.get_host("docker://" + self.pbm_cli)
        timeout = time.time() + 60
    
        while True:
            if not self.get_status()['running']:
                break
            if time.time() > timeout:
                assert False, "Cannot start restore, another operation running"
            time.sleep(1)
        Cluster.log("Restore started")
        timeout=kwargs.get('timeout', 240)
        result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
    
        if result.rc == 0:
            Cluster.log(result.stdout)
        else:
            # try to catch possible failures if timeout exceeded
            error=''
            for host in self.mongod_hosts:
                try:
                    container = docker.from_env().containers.get(host)
                    get_logs = container.exec_run(
                        'cat /var/lib/mongo/pbm.restore.log', stderr=False)
                    if get_logs.exit_code == 0:
                        Cluster.log(
                            "!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
                        logs = get_logs.output.decode('utf-8')
                        Cluster.log(logs)
                        if '"s":"F"' in logs:
                            error = logs
                except docker.errors.APIError:
                    pass
            if error:
                assert False, result.stdout + result.stderr + "\n" + error
            else:
>               assert False, result.stdout + result.stderr
E               AssertionError: Starting restore 2024-09-23T19:22:44.521484112Z from '2024-09-23T19:22:14Z'........................................................................................................................Error: no confirmation that restore has successfully started. Replsets status:
E               <no replset has started restore>

cluster.py:464: AssertionError

During handling of the above exception, another exception occurred:

start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fdbef450>

    @pytest.mark.timeout(600,func_only=True)
    def test_incremental(start_cluster,cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup = cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
        cluster.upgrade()
        cluster.check_pbm_status()
        try:
            cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)
            assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == len(documents)
            assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False)
            Cluster.log("Finished successfully")
        except AssertionError as e:
            if "is not compatible with" in str(e):
                Cluster.log("[PBM-1069] Expected failure: \n" + str(e))
                Cluster.log("Finished successfully")
            else:
>               assert False, str(e)
E               AssertionError: Starting restore 2024-09-23T19:22:44.521484112Z from '2024-09-23T19:22:14Z'........................................................................................................................Error: no confirmation that restore has successfully started. Replsets status:
E                 <no replset has started restore>
E                 
E                 
E               assert False

test_upgrade.py:97: AssertionError

Check failure on line 59 in psmdb-testing/pbm-functional/pytest/test_vault.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_vault.test_incremental_PBM_T200 

AssertionError
Raw output 
start_cluster = True, cluster = <cluster.Cluster object at 0x7f31fd7fcd10>

    @pytest.mark.timeout(300,func_only=True)
    def test_incremental_PBM_T200(start_cluster,cluster):
        cluster.check_pbm_status()
        cluster.make_backup("incremental --base")
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("incremental")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
>       cluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)

test_vault.py:59: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cluster.py:475: in make_restore
    self.make_resync()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f31fd7fcd10>

    def make_resync(self):
        n = testinfra.get_host("docker://" + self.pbm_cli)
        result = n.check_output('pbm config --force-resync --out json')
        parsed_result = json.loads(result)
        Cluster.log('Started resync: ' + result)
        timeout = time.time() + 30
        while True:
            logs = self.__find_event_msg("resync", "succeed")
            if logs:
                break
            if time.time() > timeout:
>               assert False
E               AssertionError

cluster.py:368: AssertionError
View more details on GitHub Actions