diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 37327ade9..cbad11bfb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -124,7 +124,7 @@ jobs:
           sudo apt-get install -y mingw-w64
           rustup target add x86_64-pc-windows-gnu
           cargo run -- build --no-sdist -m test-crates/pyo3-pure/Cargo.toml --target x86_64-pc-windows-gnu
-      - name: test cross compiling with PYO3_CONFIG_FILE
+      - name: test compiling with PYO3_CONFIG_FILE
         shell: bash
         run: |
           export PYO3_CONFIG_FILE=$(pwd)/test-crates/pyo3-mixed/pyo3-config.txt
diff --git a/src/build_options.rs b/src/build_options.rs
index 9c377a246..9b4cc51ad 100644
--- a/src/build_options.rs
+++ b/src/build_options.rs
@@ -656,7 +656,18 @@ pub fn find_interpreter(
     match bridge {
         BridgeModel::Bindings(binding_name, _) => {
             let mut interpreters = Vec::new();
-            if binding_name.starts_with("pyo3") && target.is_unix() && target.cross_compiling() {
+            if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") {
+                if !binding_name.starts_with("pyo3") {
+                    bail!("Only pyo3 bindings can be configured with PYO3_CONFIG_FILE");
+                }
+                let interpreter_config =
+                    InterpreterConfig::from_pyo3_config(config_file.as_ref(), target)
+                        .context("Invalid PYO3_CONFIG_FILE")?;
+                interpreters.push(PythonInterpreter::from_config(interpreter_config));
+            } else if binding_name.starts_with("pyo3")
+                && target.is_unix()
+                && target.cross_compiling()
+            {
                 if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") {
                     let host_interpreters =
                         find_host_interpreter(bridge, interpreter, target, min_python_minor)?;
@@ -724,11 +735,6 @@ pub fn find_interpreter(
                         platform: None,
                         runnable: false,
                     });
-                } else if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") {
-                    let interpreter_config =
-                        InterpreterConfig::from_pyo3_config(config_file.as_ref(), target)
-                            .context("Invalid PYO3_CONFIG_FILE")?;
-                    interpreters.push(PythonInterpreter::from_config(interpreter_config));
                 } else {
                     if interpreter.is_empty() {
                         bail!("Couldn't find any python interpreters. Please specify at least one with -i");