[air/output] Improve passed time display (ray-project#34951)

This changes the formatting of the total running time like this:

```
Trial easy_objective_b12ae_00003 completed training after 5 iterations at 2023-05-02 14:46:11. Total running time: 4d 3hr 20min, 2s

Trial easy_objective_d3d85_00008 completed training after 5 iterations at 2023-05-02 14:47:09. Total running time: 4d 0hr 20min 2s

Trial easy_objective_cbd93_00007 finished iteration 5 at 2023-05-02 14:46:56. Total running time: 20min 2s

Trial easy_objective_14e6b_00001 completed training after 5 iterations at 2023-05-02 14:48:58. Total running time: 1hr 0min 2s
```

It does not turn metrics (e.g. `time_this_iter` or `total_time_s`) into written sentences, as this will make it harder to identify these metrics for configuration purposes.

Signed-off-by: Kai Fricke <kai@anyscale.com>

python/ray/tune/experimental/output.py

@@ -111,22 +111,27 @@ def _get_time_str(start_time: float, current_time: float) -> Tuple[str, str]:
     delta: datetime.timedelta = current_time_dt - start_time_dt
 
     rest = delta.total_seconds()
-    days = rest // (60 * 60 * 24)
+    days = int(rest // (60 * 60 * 24))
 
     rest -= days * (60 * 60 * 24)
-    hours = rest // (60 * 60)
+    hours = int(rest // (60 * 60))
 
     rest -= hours * (60 * 60)
-    minutes = rest // 60
+    minutes = int(rest // 60)
 
-    seconds = rest - minutes * 60
+    seconds = int(rest - minutes * 60)
 
+    running_for_str = ""
     if days > 0:
-        running_for_str = f"{days:.0f} days, "
-    else:
-        running_for_str = ""
+        running_for_str += f"{days:d}d "
+
+    if hours > 0 or running_for_str:
+        running_for_str += f"{hours:d}hr "
+
+    if minutes > 0 or running_for_str:
+        running_for_str += f"{minutes:d}min "
 
-    running_for_str += f"{hours:02.0f}:{minutes:02.0f}:{seconds:05.2f}"
+    running_for_str += f"{seconds:d}s"
 
     return f"{current_time_dt:%Y-%m-%d %H:%M:%S}", running_for_str
 
@@ -487,8 +492,12 @@ def experiment_started(
 
     @property
     def _time_heartbeat_str(self):
-        current_time_str, running_for_str = _get_time_str(self._start_time, time.time())
-        return f"Current time: {current_time_str} " f"(running for {running_for_str})"
+        current_time_str, running_time_str = _get_time_str(
+            self._start_time, time.time()
+        )
+        return (
+            f"Current time: {current_time_str}. Total running time: " + running_time_str
+        )
 
     def print_heartbeat(self, trials, *args, force: bool = False):
         if self._verbosity < self._heartbeat_threshold:
@@ -834,11 +843,11 @@ def on_trial_result(
     ):
         if self._verbosity < self._intermediate_result_verbosity:
             return
-        curr_time, running_time = _get_time_str(self._start_time, time.time())
+        curr_time_str, running_time_str = _get_time_str(self._start_time, time.time())
         print(
             f"{self._addressing_tmpl.format(trial)} "
             f"finished iteration {result[TRAINING_ITERATION]} "
-            f"at {curr_time} (running for {running_time})."
+            f"at {curr_time_str}. Total running time: " + running_time_str
         )
         self._print_result(trial, result)
 
@@ -847,14 +856,14 @@ def on_trial_complete(
     ):
         if self._verbosity < self._start_end_verbosity:
             return
-        curr_time, running_time = _get_time_str(self._start_time, time.time())
+        curr_time_str, running_time_str = _get_time_str(self._start_time, time.time())
         finished_iter = 0
         if trial.last_result and TRAINING_ITERATION in trial.last_result:
             finished_iter = trial.last_result[TRAINING_ITERATION]
         print(
             f"{self._addressing_tmpl.format(trial)} "
             f"completed training after {finished_iter} iterations "
-            f"at {curr_time} (running for {running_time})."
+            f"at {curr_time_str}. Total running time: " + running_time_str
         )
         self._print_result(trial)
 

python/ray/tune/tests/output/test_output.py

@@ -1,7 +1,6 @@
 import pytest
 import sys
 
-import time
 from freezegun import freeze_time
 
 from ray.tune.experimental.output import (
@@ -55,8 +54,28 @@
 
 @freeze_time("Mar 27th, 2023", auto_tick_seconds=15)
 def test_get_time_str():
-    result = _get_time_str(time.time(), time.time())
-    assert result == ("2023-03-27 00:00:15", "00:00:15.00")
+    base = 1679875200  # 2023-03-27 00:00:00
+
+    assert _get_time_str(base, base) == ("2023-03-27 00:00:00", "0s")
+    assert _get_time_str(base, base + 15) == ("2023-03-27 00:00:15", "15s")
+    assert _get_time_str(base, base + 60) == ("2023-03-27 00:01:00", "1min 0s")
+    assert _get_time_str(base, base + 65) == ("2023-03-27 00:01:05", "1min 5s")
+    assert _get_time_str(base, base + 3600) == (
+        "2023-03-27 01:00:00",
+        "1hr 0min 0s",
+    )
+    assert _get_time_str(base, base + 3605) == (
+        "2023-03-27 01:00:05",
+        "1hr 0min 5s",
+    )
+    assert _get_time_str(base, base + 3660) == (
+        "2023-03-27 01:01:00",
+        "1hr 1min 0s",
+    )
+    assert _get_time_str(base, base + 86400) == (
+        "2023-03-28 00:00:00",
+        "1d 0hr 0min 0s",
+    )
 
 
 def test_get_trials_by_state():