start_cluster = True, cluster = <cluster.Cluster object at 0x7f6422ec1e90>

    @pytest.mark.timeout(300, func_only=True)
    def test_logical_PBM_T266(start_cluster, cluster):
        cluster.check_pbm_status()
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        result = cluster.exec_pbm_cli("backup --wait")
>       assert result.rc != 0, result.stdout + result.stderr
E       AssertionError: Starting backup '2024-09-23T18:32:28Z'....
E         Waiting for '2024-09-23T18:32:28Z' backup........................ done
E         Backup '2024-09-23T18:32:28Z' to remote store 'http://nginx-minio:21114/bcp/pbme2etest'
E         
E       assert 0 != 0
E        +  where 0 = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7f6422be51d0>, exit_status=0, command=b'pbm ........... done\nBackup '2024-09-23T18:32:28Z' to remote store 'http://nginx-minio:21114/bcp/pbme2etest'\n", _stderr=b'').rc

test_PBM-1114.py:56: AssertionError

start_cluster = True, cluster = <cluster.Cluster object at 0x7f641cafe850>

    @pytest.mark.timeout(600,func_only=True)
    def test_logical(start_cluster,cluster):
        cluster.check_pbm_status()
        pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
        backup=cluster.make_backup("logical")
        result=pymongo.MongoClient(cluster.connection)["test"]["test"].delete_many({})
        assert int(result.deleted_count) == len(documents)
        cluster.upgrade()
        cluster.check_pbm_status()
>       cluster.make_restore(backup,check_pbm_status=True)

test_upgrade.py:58: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7f641cafe850>, name = '2024-09-23T19:31:34Z'
kwargs = {'check_pbm_status': True}
client = MongoClient(host=['mongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7f6422be51d0>, exit_status=1, command=b'time...0770952Z from '2024-09-23T19:31:34Z'...............", _stderr=b'Error: get metadata: get: context deadline exceeded\n')
n = <testinfra.host.Host docker://rscfg01>, timeout = 240, error = ''
host = 'rscfg03', container = <Container: 5c1f92b4079b>

    def make_restore(self, name, **kwargs):
        if self.layout == "sharded":
            client = pymongo.MongoClient(self.connection)
            result = client.admin.command("balancerStop")
            client.close()
            Cluster.log("Stopping balancer: " + str(result))
            self.stop_mongos()
        self.stop_arbiters()
        n = testinfra.get_host("docker://" + self.pbm_cli)
        timeout = time.time() + 60
    
        while True:
            if not self.get_status()['running']:
                break
            if time.time() > timeout:
                assert False, "Cannot start restore, another operation running"
            time.sleep(1)
        Cluster.log("Restore started")
        timeout=kwargs.get('timeout', 240)
        result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
    
        if result.rc == 0:
            Cluster.log(result.stdout)
        else:
            # try to catch possible failures if timeout exceeded
            error=''
            for host in self.mongod_hosts:
                try:
                    container = docker.from_env().containers.get(host)
                    get_logs = container.exec_run(
                        'cat /var/lib/mongo/pbm.restore.log', stderr=False)
                    if get_logs.exit_code == 0:
                        Cluster.log(
                            "!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
                        logs = get_logs.output.decode('utf-8')
                        Cluster.log(logs)
                        if '"s":"F"' in logs:
                            error = logs
                except docker.errors.APIError:
                    pass
            if error:
                assert False, result.stdout + result.stderr + "\n" + error
            else:
>               assert False, result.stdout + result.stderr
E               AssertionError: Starting restore 2024-09-23T19:32:06.760770952Z from '2024-09-23T19:31:34Z'...............Error: get metadata: get: context deadline exceeded

cluster.py:464: AssertionError