Re-introduce retries in the execution status test to further reduce f…

…lakiness caused by race conditions

tests/services/kernels/test_execution_state.py

@@ -13,6 +13,10 @@
 from tornado.httpclient import HTTPClientError
 from traitlets.config import Config
 
+MAX_POLL_ATTEMPTS = 10
+POLL_INTERVAL = 1
+MINIMUM_CONSISTENT_COUNT = 4
+
 
 @flaky
 async def test_execution_state(jp_fetch, jp_ws_fetch):
@@ -98,10 +102,30 @@ async def test_execution_state(jp_fetch, jp_ws_fetch):
 
 
 async def get_execution_state(kid, jp_fetch):
-    r = await jp_fetch("api", "kernels", kid, method="GET")
-    model = json.loads(r.body.decode())
-    execution_state = model["execution_state"]
-    return execution_state
+    # There is an inherent race condition when getting the kernel execution status
+    # where we might fetch the status right before an expected state change occurs.
+    #
+    # To work-around this, we don't return the status until we've been able to fetch
+    # it twice in a row and get the same result both times.
+    last_execution_states = []
+
+    for _ in range(MAX_POLL_ATTEMPTS):
+        r = await jp_fetch("api", "kernels", kid, method="GET")
+        model = json.loads(r.body.decode())
+        execution_state = model["execution_state"]
+        last_execution_states.append(execution_state)
+        consistent_count = 0
+        last_execution_state = None
+        for es in last_execution_states:
+            if es != last_execution_state:
+                consistent_count = 0
+                last_execution_state = es
+            consistent_count += 1
+            if consistent_count >= MINIMUM_CONSISTENT_COUNT:
+                return es
+        time.sleep(POLL_INTERVAL)
+
+    raise AssertionError("failed to get a consistent execution state")
 
 
 async def poll_for_parent_message_status(kid, parent_message_id, target_status, ws):