From 6a08df612feb44ba7b7bf1fd318a69e266f090e7 Mon Sep 17 00:00:00 2001
From: Ross Wightman <rwightman@gmail.com>
Date: Thu, 7 Nov 2024 21:45:46 -0800
Subject: [PATCH] Change eps defaults in adafactor_bv again after some checking

---
 timm/optim/adafactor_bv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/timm/optim/adafactor_bv.py b/timm/optim/adafactor_bv.py
index d603b528..bad982ee 100644
--- a/timm/optim/adafactor_bv.py
+++ b/timm/optim/adafactor_bv.py
@@ -214,8 +214,8 @@ def _single_tensor_adafactor(
         exp_avg = exp_avgs[i]
         step_t = state_steps[i]
         if eps is None:
-            # use square of machine eps for grad dtype if not set
-            eps = torch.finfo(grad.dtype).eps ** 2
+            # default eps for avoiding div by zero, diff from float type eps
+            eps = 1e-7 if grad.dtype == torch.float16 else 1e-30
 
         # Update step
         step_t += 1