Merge pull request #244 from sony/feature/20200921-extend-watchdog-time

extend watch dog time so not too sensitive
sony · Sep 29, 2020 · cf74f5e · cf74f5e
2 parents cf53048 + 85bf3f6
commit cf74f5e
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/include/nbla/cuda/communicator/multi_process_data_parallel_communicator.hpp b/include/nbla/cuda/communicator/multi_process_data_parallel_communicator.hpp
@@ -60,7 +60,7 @@ template <typename T>
 class NBLA_API MultiProcessDataParallelCommunicatorNccl
     : public MultiProcessDataParallelCommunicator<T> {
 protected:
-  int all_reduce_timeout_ = 100; // timeout=10s
+  int all_reduce_timeout_ = 500; // timeout=50s
   Watchdog watch_dog_;
   int device_id_;
 

diff --git a/include/nbla/cuda/communicator/watch_dog.hpp b/include/nbla/cuda/communicator/watch_dog.hpp
@@ -19,7 +19,7 @@ namespace nbla {
 
 class Watchdog {
 private:
-  static const int TIMEOUT_TICKS = 100;    // default timeout is 10s
+  static const int TIMEOUT_TICKS = 600;    // default timeout is 60s
   static const int TICK = 100;             // 100 ms each tick
   static const int STOP_WATCH_DOG = -1000; // arbitrary negative value
   static const int EXIT_WATCH_DOG = 1;     // exit flag