diff --git a/.travis.yml b/.travis.yml
index 8ede978074a9c..e97994262e825 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,20 +31,13 @@ env:
 git:
   depth: false
 
-matrix:
+jobs:
   fast_finish: true
 
   include:
     - arch: arm64
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
-  allow_failures:
-    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
+        - JOB="3.7, arm64" PYTEST_WORKERS=4 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
 before_install:
   - echo "before_install"
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 593939431d5eb..b50b1c104b4af 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -20,7 +20,11 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
+# With --dist=no, pytest distributs one test at a time.
+# If using other options, running the test suite would be extremely slow on arm64 machine.
+# See https://pypi.org/project/pytest-xdist/ for details.
+# GH 36719
+PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=no -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
 
 if [[ $(uname) != "Linux"  && $(uname) != "Darwin" ]]; then
     # GH#37455 windows py38 build appears to be running out of memory
diff --git a/ci/setup_env.sh b/ci/setup_env.sh
index c36422884f2ec..6ad4b85aba2a6 100755
--- a/ci/setup_env.sh
+++ b/ci/setup_env.sh
@@ -37,7 +37,7 @@ else
 fi
 
 if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
-  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh"
+  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.9.2-5/Miniforge3-4.9.2-5-Linux-aarch64.sh"
 else
   CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"
 fi
diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py
index eb6cf4f9d7d85..c972cf3d39b51 100644
--- a/pandas/compat/__init__.py
+++ b/pandas/compat/__init__.py
@@ -26,6 +26,7 @@
 PY39 = sys.version_info >= (3, 9)
 PYPY = platform.python_implementation() == "PyPy"
 IS64 = sys.maxsize > 2 ** 32
+ARM64 = platform.machine() == "arm64" or platform.machine() == "aarch64"
 
 
 def set_function_name(f: F, name: str, cls) -> F:
diff --git a/pandas/tests/indexes/interval/test_astype.py b/pandas/tests/indexes/interval/test_astype.py
index c269d6ff11896..180469be3fe2f 100644
--- a/pandas/tests/indexes/interval/test_astype.py
+++ b/pandas/tests/indexes/interval/test_astype.py
@@ -3,6 +3,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat import ARM64
+
 from pandas.core.dtypes.dtypes import CategoricalDtype, IntervalDtype
 
 from pandas import (
@@ -165,6 +167,7 @@ def test_subtype_integer_with_non_integer_borders(self, subtype):
         )
         tm.assert_index_equal(result, expected)
 
+    @pytest.mark.xfail(ARM64, reason="GH 38923")
     def test_subtype_integer_errors(self):
         # float64 -> uint64 fails with negative values
         index = interval_range(-10.0, 10.0)
diff --git a/pandas/tests/tools/test_to_numeric.py b/pandas/tests/tools/test_to_numeric.py
index d5b4bda35ca2b..db3debc4c5524 100644
--- a/pandas/tests/tools/test_to_numeric.py
+++ b/pandas/tests/tools/test_to_numeric.py
@@ -4,6 +4,8 @@
 from numpy import iinfo
 import pytest
 
+from pandas.compat import ARM64
+
 import pandas as pd
 from pandas import DataFrame, Index, Series, to_numeric
 import pandas._testing as tm
@@ -747,7 +749,7 @@ def test_to_numeric_from_nullable_string(values, expected):
             "UInt64",
             "signed",
             "UInt64",
-            marks=pytest.mark.xfail(reason="GH38798"),
+            marks=pytest.mark.xfail(not ARM64, reason="GH38798"),
         ),
         ([1, 1], "Int64", "unsigned", "UInt8"),
         ([1.0, 1.0], "Float32", "unsigned", "UInt8"),
diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py
index b275b64ff706b..71c0945c4b37f 100644
--- a/pandas/tests/window/test_rolling.py
+++ b/pandas/tests/window/test_rolling.py
@@ -3,6 +3,7 @@
 import numpy as np
 import pytest
 
+from pandas.compat import ARM64
 from pandas.errors import UnsupportedFunctionCall
 
 from pandas import (
@@ -891,6 +892,7 @@ def test_rolling_sem(frame_or_series):
     tm.assert_series_equal(result, expected)
 
 
+@pytest.mark.xfail(ARM64, reason="GH 38921")
 @pytest.mark.parametrize(
     ("func", "third_value", "values"),
     [