more debugging

NVIDIA · Apr 3, 2020 · a0bf956 · a0bf956
1 parent feb93a2
commit a0bf956
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/apex/contrib/optimizers/fused_lamb.py b/apex/contrib/optimizers/fused_lamb.py
@@ -83,6 +83,7 @@ def __init__(self, params, lr=1e-3, bias_correction=True,
 
         self.adam_w_mode = 1 if adam_w_mode else 0
         self.set_grad_none = set_grad_none
+        print("debugging LAMB")
 
     def zero_grad(self):
         if self.set_grad_none:
@@ -130,6 +131,8 @@ def step(self, closure=None):
         # blend two grad norms to get global grad norm
         global_grad_norm = math.sqrt(g_norm_32 * g_norm_32 + g_norm_16 * g_norm_16)
         max_grad_norm = self.defaults['max_grad_norm']
+        print("====global_grad_norm:", global_grad_norm)
+        print("====max_grad_norm:", max_grad_norm)
 
         for group in self.param_groups:
             bias_correction = 1 if group['bias_correction'] else 0