diff --git a/docker/mongodb-kubernetes-tests/kubetester/mongodb_common.py b/docker/mongodb-kubernetes-tests/kubetester/mongodb_common.py index f032ed6c1..59d623e90 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/mongodb_common.py +++ b/docker/mongodb-kubernetes-tests/kubetester/mongodb_common.py @@ -11,12 +11,13 @@ class MongoDBCommon: @TRACER.start_as_current_span("wait_for") - def wait_for(self, fn, timeout=None, should_raise=True): + def wait_for(self, fn, timeout=None, should_raise=True, persist_for=1): if timeout is None: timeout = 600 initial_timeout = timeout wait = 3 + retries = 0 while timeout > 0: try: self.reload() @@ -24,7 +25,12 @@ def wait_for(self, fn, timeout=None, should_raise=True): print(f"Caught error: {e} while waiting for {fn.__name__}") pass if fn(self): - return True + retries += 1 + if retries == persist_for: + return True + else: + retries = 0 + timeout -= wait time.sleep(wait) diff --git a/docker/mongodb-kubernetes-tests/kubetester/opsmanager.py b/docker/mongodb-kubernetes-tests/kubetester/opsmanager.py index 0a463c839..1ce4a4d2b 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/opsmanager.py +++ b/docker/mongodb-kubernetes-tests/kubetester/opsmanager.py @@ -1017,13 +1017,7 @@ def is_om_multi_cluster(self): return self["spec"].get("topology", "") == "MultiCluster" class StatusCommon: - def assert_reaches_phase( - self, - phase: Phase, - msg_regexp=None, - timeout=None, - ignore_errors=False, - ): + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=None, ignore_errors=False, persist_for=1): intermediate_events = ( # This can be an intermediate error, right before we check for this secret we create it. # The cluster might just be slow @@ -1046,6 +1040,7 @@ def assert_reaches_phase( ), timeout, should_raise=True, + persist_for=persist_for, ) end_time = time.time() span = trace.get_current_span() @@ -1110,8 +1105,8 @@ def __init__(self, ops_manager: MongoDBOpsManager): def assert_abandons_phase(self, phase: Phase, timeout=400): super().assert_abandons_phase(phase, timeout) - def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=1000, ignore_errors=False): - super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors) + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=1000, ignore_errors=False, persist_for=1): + super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors, persist_for=persist_for) def get_phase(self) -> Optional[Phase]: try: @@ -1156,8 +1151,8 @@ def __init__(self, ops_manager: MongoDBOpsManager): def assert_abandons_phase(self, phase: Phase, timeout=400): super().assert_abandons_phase(phase, timeout) - def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=1200, ignore_errors=False): - super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors) + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=1200, ignore_errors=False, persist_for=1): + super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors, persist_for=persist_for) def get_phase(self) -> Optional[Phase]: try: diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_upgrade_downgrade_v1_27_to_mck.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_upgrade_downgrade_v1_27_to_mck.py index d3deeaa79..6e04a50ea 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_upgrade_downgrade_v1_27_to_mck.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_upgrade_downgrade_v1_27_to_mck.py @@ -256,7 +256,7 @@ def test_scale_appdb(self, ops_manager: MongoDBOpsManager): # Reordering the clusters triggers a change in the state ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = scale_on_upgrade.cluster_spec ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=500) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=250) def test_migrated_state_correctness( diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py b/docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py index 1fa4fb48c..308df6ca9 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py @@ -104,7 +104,7 @@ def test_running(namespace: str): try: logger.debug(f"Waiting for {sc.name} to reach Running phase") # Once the first resource reached Running, it shouldn't take more than ~300s for the others to do so - sc.assert_reaches_phase(Phase.Running, timeout=900 if first_iter else 300) + sc.assert_reaches_phase(Phase.Running, timeout=1200 if first_iter else 300) succeeded_resources.append(sc.name) first_iter = False logger.info(f"{sc.name} reached Running phase") diff --git a/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py b/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py index 939c69afe..3c3d8adbd 100644 --- a/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py +++ b/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py @@ -161,8 +161,8 @@ def test_upgrade_operator( @mark.e2e_appdb_tls_operator_upgrade_v1_32_to_mck def test_om_tls_ok(ops_manager_tls: MongoDBOpsManager): ops_manager_tls.load() - ops_manager_tls.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) - ops_manager_tls.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_tls.appdb_status().assert_reaches_phase(Phase.Running, timeout=900, persist_for=3) + ops_manager_tls.om_status().assert_reaches_phase(Phase.Running, timeout=900, persist_for=3) ops_manager_tls.get_om_tester().assert_healthiness()