diff --git a/gvfs.c b/gvfs.c
index 7235199c04ac39..4360d77075ea0f 100644
--- a/gvfs.c
+++ b/gvfs.c
@@ -14,9 +14,6 @@ static int early_core_gvfs_config(const char *var, const char *value, void *data
 
 void gvfs_load_config_value(const char *value)
 {
-	if (gvfs_config_loaded)
-		return;
-
 	if (value)
 		core_gvfs = git_config_bool_or_int("core.gvfs", value, &core_gvfs_is_bool);
 	else if (startup_info->have_repository == 0)
@@ -27,12 +24,13 @@ void gvfs_load_config_value(const char *value)
 	/* Turn on all bits if a bool was set in the settings */
 	if (core_gvfs_is_bool && core_gvfs)
 		core_gvfs = -1;
-
-	gvfs_config_loaded = 1;
 }
 
 int gvfs_config_is_set(int mask)
 {
-	gvfs_load_config_value(0);
+	if (!gvfs_config_loaded)
+		gvfs_load_config_value(0);
+
+	gvfs_config_loaded = 1;
 	return (core_gvfs & mask) == mask;
 }
diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
index 8e56ec461d7ca9..777d4f19985cc6 100755
--- a/t/t0021-conversion.sh
+++ b/t/t0021-conversion.sh
@@ -324,6 +324,10 @@ test_expect_success "filter: smudge filters blocked when under GVFS" '
 	test_config filter.empty-in-repo.smudge "echo smudged && cat" &&
 	test_config core.gvfs 64 &&
 
+	test_must_fail git checkout &&
+
+	# ensure the local core.gvfs setting overwrites the global setting
+	git config --global core.gvfs false &&
 	test_must_fail git checkout
 '