diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c
index 3bcdb28b636611..f3e96191eb6f35 100644
--- a/drivers/net/phy/at803x.c
+++ b/drivers/net/phy/at803x.c
@@ -103,6 +103,18 @@ static int at803x_debug_reg_mask(struct phy_device *phydev, u16 reg,
 	return phy_write(phydev, AT803X_DEBUG_DATA, val);
 }
 
+static int at803x_enable_rx_delay(struct phy_device *phydev)
+{
+	return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_0, 0,
+				     AT803X_DEBUG_RX_CLK_DLY_EN);
+}
+
+static int at803x_enable_tx_delay(struct phy_device *phydev)
+{
+	return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_5, 0,
+				     AT803X_DEBUG_TX_CLK_DLY_EN);
+}
+
 static int at803x_disable_rx_delay(struct phy_device *phydev)
 {
 	return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_0,
@@ -241,23 +253,42 @@ static int at803x_config_init(struct phy_device *phydev)
 	if (ret < 0)
 		return ret;
 
-	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID ||
-			phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
-			phydev->interface == PHY_INTERFACE_MODE_RGMII) {
-		ret = at803x_disable_rx_delay(phydev);
+	/* The RX and TX delay default is:
+	 *   after HW reset: RX delay enabled and TX delay disabled
+	 *   after SW reset: RX delay enabled, while TX delay retains the
+	 *   value before reset.
+	 *
+	 * So let's first disable the RX and TX delays in PHY and enable
+	 * them based on the mode selected (this also takes care of RGMII
+	 * mode where we expect delays to be disabled)
+	 */
+
+	ret = at803x_disable_rx_delay(phydev);
+	if (ret < 0)
+		return ret;
+	ret = at803x_disable_tx_delay(phydev);
+	if (ret < 0)
+		return ret;
+
+	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
+	    phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID) {
+		/* If RGMII_ID or RGMII_RXID are specified enable RX delay,
+		 * otherwise keep it disabled
+		 */
+		ret = at803x_enable_rx_delay(phydev);
 		if (ret < 0)
 			return ret;
 	}
 
-	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID ||
-			phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
-			phydev->interface == PHY_INTERFACE_MODE_RGMII) {
-		ret = at803x_disable_tx_delay(phydev);
-		if (ret < 0)
-			return ret;
+	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
+	    phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID) {
+		/* If RGMII_ID or RGMII_TXID are specified enable TX delay,
+		 * otherwise keep it disabled
+		 */
+		ret = at803x_enable_tx_delay(phydev);
 	}
 
-	return 0;
+	return ret;
 }
 
 static int at803x_ack_interrupt(struct phy_device *phydev)