[DCP] Improve with_temp_dir robustness (pytorch#111106)

Calling os.sync() to ensure the tempfile can be seens across ranks.

Differential Revision: [D50209697](https://our.internmc.facebook.com/intern/diff/D50209697/)
Pull Request resolved: pytorch#111106
Approved by: https://github.com/Skylion007, https://github.com/wz337

torch/testing/_internal/distributed/checkpoint_utils.py

@@ -1,5 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates
 
+import os
 import shutil
 import tempfile
 from functools import wraps
@@ -27,8 +28,10 @@ def wrapper(self, *args: Tuple[object], **kwargs: Dict[str, Any]) -> None:
         object_list = [temp_dir]
 
         # Broadcast temp_dir to all the other ranks
+        os.sync()
         dist.broadcast_object_list(object_list)
         self.temp_dir = object_list[0]
+        os.sync()
 
         try:
             func(self, *args, **kwargs)