diff --git a/csrc/deepspeed4science/evoformer_attn/attention.cu b/csrc/deepspeed4science/evoformer_attn/attention_cu.cu
similarity index 100%
rename from csrc/deepspeed4science/evoformer_attn/attention.cu
rename to csrc/deepspeed4science/evoformer_attn/attention_cu.cu
diff --git a/deepspeed/comm/ccl.py b/deepspeed/comm/ccl.py
index 6e915d0ca430..38a22f20109f 100644
--- a/deepspeed/comm/ccl.py
+++ b/deepspeed/comm/ccl.py
@@ -170,7 +170,7 @@ def get_all_ranks_from_group(self, group):
             while True:
                 results.append(super(CCLBackend, self).get_global_rank(group, rank))
                 rank += 1
-        except ValueError:
+        except (ValueError, RuntimeError):
             pass
         if tuple(results) not in self.groups:
             self._new_group(results, group)
diff --git a/deepspeed/runtime/pipe/engine.py b/deepspeed/runtime/pipe/engine.py
index 2542dd602b1f..b89c1aca7f2a 100644
--- a/deepspeed/runtime/pipe/engine.py
+++ b/deepspeed/runtime/pipe/engine.py
@@ -988,7 +988,7 @@ def _exec_send_grads(self, buffer_id):
             if isinstance(inputs, tuple):
                 first_input = inputs[0]
                 assert all([torch.is_tensor(elt) for elt in inputs[1:]])
-                inputs_grad_tail = [elt.grad for elt in inputs[1:] if elt.grad is not None]
+                inputs_grad_tail = [elt.grad for elt in inputs[1:]]
             elif torch.is_tensor(inputs):
                 first_input = inputs
                 inputs_grad_tail = []
diff --git a/op_builder/builder.py b/op_builder/builder.py
index 1fdcd485f5fb..62683774f20a 100644
--- a/op_builder/builder.py
+++ b/op_builder/builder.py
@@ -486,6 +486,9 @@ def jit_load(self, verbose=True):
                 cxx_args.append("-DBF16_AVAILABLE")
                 nvcc_args.append("-DBF16_AVAILABLE")
 
+        if self.is_rocm_pytorch():
+            cxx_args.append("-D__HIP_PLATFORM_AMD__=1")
+
         op_module = load(name=self.name,
                          sources=self.strip_empty_entries(sources),
                          extra_include_paths=self.strip_empty_entries(extra_include_paths),
diff --git a/op_builder/evoformer_attn.py b/op_builder/evoformer_attn.py
index f4311848d0d4..6e7721f94e01 100644
--- a/op_builder/evoformer_attn.py
+++ b/op_builder/evoformer_attn.py
@@ -27,7 +27,7 @@ def extra_ldflags(self):
 
     def sources(self):
         src_dir = 'csrc/deepspeed4science/evoformer_attn'
-        return [f'{src_dir}/attention.cpp', f'{src_dir}/attention_back.cu', f'{src_dir}/attention.cu']
+        return [f'{src_dir}/attention.cpp', f'{src_dir}/attention_back.cu', f'{src_dir}/attention_cu.cu']
 
     def nvcc_args(self):
         args = super().nvcc_args()