diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 39e8b8e55a7..86fc0710085 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -404,7 +404,7 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.x"
+          - "3.11"
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -419,6 +419,8 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: "lib/py/setup.py"
 
       - name: Python setup
         run: |
@@ -430,7 +432,6 @@ jobs:
         run: ./bootstrap.sh
 
       - name: Run configure 3.x
-        if: matrix.python-version == '3.x'
         run: |
           ./configure $(echo $CONFIG_ARGS_FOR_LIBS | sed 's/without-py3/with-py3/')
 
@@ -448,7 +449,7 @@ jobs:
         run: make -C lib/py
 
       - name: Run make install for python
-        run: sudo make -C lib/py install
+        run: make -C lib/py install
 
       # - name: Run make install-exec-hook for python
       #   run: sudo make -C lib/py install-exec-hook