Fix flaky NCCL error handling tests. (#42149)

pritamdamania · facebook-github-bot · commit 8deb4fe809ca · 2020-07-28T18:38:26.000-07:00
Summary: Pull Request resolved: #42149 Some of these tests were flaky since we could kill the process in some way without cleaning up the ProcessGroup. This resulted in issues where the FileStore didn't clean up appropriately resulting in other processes in the group to crash. Fixed this by explicitly deleting the process_group before we bring a process down forcibly. ghstack-source-id: 108629057 Test Plan: waitforbuildbot Reviewed By: mrshenli Differential Revision: D22785042 fbshipit-source-id: c31d0f723badbc23b7258e322f75b57e0a1a42cf
diff --git a/test/distributed/test_c10d.py b/test/distributed/test_c10d.py
@@ -3413,6 +3413,8 @@ def _test_nccl_errors_blocking(self, func):
             # aborting nccl communicators before throwing Operation timed out
             a = torch.rand(10).cuda(self.rank)
         elif self.rank == 1:
+            # Clean up structures (ex: files for FileStore before going down)
+            del process_group
             func()
         else:
             # Wait for timeout
@@ -3494,7 +3496,7 @@ def _wait_for_comm_abort(self, process_group):
                     return
                 else:
                     raise e
-            time.sleep(1)
+            time.sleep(0.1)
 
     @requires_nccl()
     @skip_if_lt_x_gpu(3)
