diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index bafb37775cbb1..2101893d39dc9 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -33,6 +33,7 @@
 from pandas.util._decorators import Appender, cache_readonly, doc
 
 from pandas.core.dtypes.cast import (
+    astype_nansafe,
     find_common_type,
     maybe_cast_to_integer_array,
     maybe_promote,
@@ -693,22 +694,21 @@ def astype(self, dtype, copy=True):
         if is_dtype_equal(self.dtype, dtype):
             return self.copy() if copy else self
 
-        elif is_categorical_dtype(dtype):
-            from pandas.core.indexes.category import CategoricalIndex
-
-            return CategoricalIndex(
-                self._values, name=self.name, dtype=dtype, copy=copy
+        if needs_i8_conversion(dtype) and is_float_dtype(self.dtype):
+            # We can't put this into astype_nansafe bc astype_nansafe allows
+            #  casting np.nan to NaT
+            raise TypeError(
+                f"Cannot convert {type(self).__name__} to dtype {dtype}; integer "
+                "values are required for conversion"
             )
 
-        elif is_extension_array_dtype(dtype):
-            return Index(np.asarray(self), name=self.name, dtype=dtype, copy=copy)
-
         try:
-            casted = self._values.astype(dtype, copy=copy)
-        except (TypeError, ValueError) as err:
+            casted = astype_nansafe(self._values, dtype=dtype, copy=True)
+        except TypeError as err:
             raise TypeError(
                 f"Cannot cast {type(self).__name__} to dtype {dtype}"
             ) from err
+
         return Index(casted, name=self.name, dtype=dtype)
 
     _index_shared_docs[
diff --git a/pandas/core/indexes/numeric.py b/pandas/core/indexes/numeric.py
index 91d27d9922aa5..d6f91c9a06739 100644
--- a/pandas/core/indexes/numeric.py
+++ b/pandas/core/indexes/numeric.py
@@ -7,12 +7,10 @@
 from pandas._typing import Dtype, DtypeObj, Label
 from pandas.util._decorators import doc
 
-from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.common import (
     is_bool,
     is_bool_dtype,
     is_dtype_equal,
-    is_extension_array_dtype,
     is_float,
     is_float_dtype,
     is_integer_dtype,
@@ -21,8 +19,6 @@
     is_scalar,
     is_signed_integer_dtype,
     is_unsigned_integer_dtype,
-    needs_i8_conversion,
-    pandas_dtype,
 )
 from pandas.core.dtypes.generic import ABCSeries
 from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna
@@ -332,21 +328,6 @@ def inferred_type(self) -> str:
         """
         return "floating"
 
-    @doc(Index.astype)
-    def astype(self, dtype, copy=True):
-        dtype = pandas_dtype(dtype)
-        if needs_i8_conversion(dtype):
-            raise TypeError(
-                f"Cannot convert Float64Index to dtype {dtype}; integer "
-                "values are required for conversion"
-            )
-        elif is_integer_dtype(dtype) and not is_extension_array_dtype(dtype):
-            # TODO(jreback); this can change once we have an EA Index type
-            # GH 13149
-            arr = astype_nansafe(self._values, dtype=dtype)
-            return Int64Index(arr, name=self.name)
-        return super().astype(dtype, copy=copy)
-
     # ----------------------------------------------------------------
     # Indexing Methods