diff --git a/src/Microsoft.AspNetCore.Authentication.Abstractions/AuthenticationOptions.cs b/src/Microsoft.AspNetCore.Authentication.Abstractions/AuthenticationOptions.cs
index 9e1df2b4..4322dbeb 100644
--- a/src/Microsoft.AspNetCore.Authentication.Abstractions/AuthenticationOptions.cs
+++ b/src/Microsoft.AspNetCore.Authentication.Abstractions/AuthenticationOptions.cs
@@ -60,6 +60,11 @@ public void AddScheme<THandler>(string name, string displayName) where THandler
                 b.HandlerType = typeof(THandler);
             });
 
+        /// <summary>
+        /// Used by as the fallback default scheme for all the other defaults."/>.
+        /// </summary>
+        public string DefaultScheme { get; set; }
+
         /// <summary>
         /// Used by as the default scheme by <see cref="IAuthenticationService.AuthenticateAsync(HttpContext, string)"/>.
         /// </summary>
diff --git a/src/Microsoft.AspNetCore.Authentication.Abstractions/IAuthenticationSchemeProvider.cs b/src/Microsoft.AspNetCore.Authentication.Abstractions/IAuthenticationSchemeProvider.cs
index 675cf52c..3d2584fc 100644
--- a/src/Microsoft.AspNetCore.Authentication.Abstractions/IAuthenticationSchemeProvider.cs
+++ b/src/Microsoft.AspNetCore.Authentication.Abstractions/IAuthenticationSchemeProvider.cs
@@ -28,7 +28,7 @@ public interface IAuthenticationSchemeProvider
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.AuthenticateAsync(HttpContext, string)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultAuthenticateScheme"/>.
-        /// Otherwise, if only a single scheme exists, that will be used, if more than one exists, null will be returned.
+        /// Otherwise, this will fallback to <see cref="AuthenticationOptions.DefaultScheme"/>.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.AuthenticateAsync(HttpContext, string)"/>.</returns>
         Task<AuthenticationScheme> GetDefaultAuthenticateSchemeAsync();
@@ -36,7 +36,7 @@ public interface IAuthenticationSchemeProvider
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.ChallengeAsync(HttpContext, string, AuthenticationProperties)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultChallengeScheme"/>.
-        /// Otherwise, if only a single scheme exists, that will be used, if more than one exists, null will be returned.
+        /// Otherwise, this will fallback to <see cref="AuthenticationOptions.DefaultScheme"/>.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.ChallengeAsync(HttpContext, string, AuthenticationProperties)"/>.</returns>
         Task<AuthenticationScheme> GetDefaultChallengeSchemeAsync();
@@ -52,7 +52,7 @@ public interface IAuthenticationSchemeProvider
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.SignInAsync(HttpContext, string, System.Security.Claims.ClaimsPrincipal, AuthenticationProperties)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultSignInScheme"/>.
-        /// Otherwise, if only a single scheme exists, that will be used, if more than one exists, null will be returned.
+        /// Otherwise, this will fallback to <see cref="AuthenticationOptions.DefaultScheme"/>.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.SignInAsync(HttpContext, string, System.Security.Claims.ClaimsPrincipal, AuthenticationProperties)"/>.</returns>
         Task<AuthenticationScheme> GetDefaultSignInSchemeAsync();
diff --git a/src/Microsoft.AspNetCore.Authentication.Core/AuthenticationSchemeProvider.cs b/src/Microsoft.AspNetCore.Authentication.Core/AuthenticationSchemeProvider.cs
index be02a5c3..f5ec8e15 100644
--- a/src/Microsoft.AspNetCore.Authentication.Core/AuthenticationSchemeProvider.cs
+++ b/src/Microsoft.AspNetCore.Authentication.Core/AuthenticationSchemeProvider.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Options;
@@ -35,42 +34,33 @@ public AuthenticationSchemeProvider(IOptions<AuthenticationOptions> options)
 
         private IDictionary<string, AuthenticationScheme> _map = new Dictionary<string, AuthenticationScheme>(StringComparer.Ordinal);
         private List<AuthenticationScheme> _requestHandlers = new List<AuthenticationScheme>();
-        private List<AuthenticationScheme> _signOutHandlers = new List<AuthenticationScheme>();
-        private List<AuthenticationScheme> _signInHandlers = new List<AuthenticationScheme>();
+
+        private Task<AuthenticationScheme> GetDefaultSchemeAsync()
+            => _options.DefaultScheme != null
+            ? GetSchemeAsync(_options.DefaultScheme)
+            : Task.FromResult<AuthenticationScheme>(null);
 
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.AuthenticateAsync(HttpContext, string)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultAuthenticateScheme"/>.
-        /// Otherwise, if only a single scheme exists, that will be used, if more than one exists, null will be returned.
+        /// Otherwise, this will fallback to <see cref="AuthenticationOptions.DefaultScheme"/>.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.AuthenticateAsync(HttpContext, string)"/>.</returns>
         public virtual Task<AuthenticationScheme> GetDefaultAuthenticateSchemeAsync()
-        {
-            if (_options.DefaultAuthenticateScheme != null)
-            {
-                return GetSchemeAsync(_options.DefaultAuthenticateScheme);
-            }
-            if (_map.Count == 1)
-            {
-                return Task.FromResult(_map.Values.First());
-            }
-            return Task.FromResult<AuthenticationScheme>(null);
-        }
+            => _options.DefaultAuthenticateScheme != null
+            ? GetSchemeAsync(_options.DefaultAuthenticateScheme)
+            : GetDefaultSchemeAsync();
 
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.ChallengeAsync(HttpContext, string, AuthenticationProperties)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultChallengeScheme"/>.
-        /// Otherwise, this will fallback to <see cref="GetDefaultAuthenticateSchemeAsync"/>.
+        /// Otherwise, this will fallback to <see cref="AuthenticationOptions.DefaultScheme"/>.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.ChallengeAsync(HttpContext, string, AuthenticationProperties)"/>.</returns>
         public virtual Task<AuthenticationScheme> GetDefaultChallengeSchemeAsync()
-        {
-            if (_options.DefaultChallengeScheme != null)
-            {
-                return GetSchemeAsync(_options.DefaultChallengeScheme);
-            }
-            return GetDefaultAuthenticateSchemeAsync();
-        }
+            => _options.DefaultChallengeScheme != null
+            ? GetSchemeAsync(_options.DefaultChallengeScheme)
+            : GetDefaultSchemeAsync();
 
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.ForbidAsync(HttpContext, string, AuthenticationProperties)"/>.
@@ -79,53 +69,31 @@ public virtual Task<AuthenticationScheme> GetDefaultChallengeSchemeAsync()
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.ForbidAsync(HttpContext, string, AuthenticationProperties)"/>.</returns>
         public virtual Task<AuthenticationScheme> GetDefaultForbidSchemeAsync()
-        {
-            if (_options.DefaultForbidScheme != null)
-            {
-                return GetSchemeAsync(_options.DefaultForbidScheme);
-            }
-            return GetDefaultChallengeSchemeAsync();
-        }
+            => _options.DefaultForbidScheme != null
+            ? GetSchemeAsync(_options.DefaultForbidScheme)
+            : GetDefaultChallengeSchemeAsync();
 
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.SignInAsync(HttpContext, string, System.Security.Claims.ClaimsPrincipal, AuthenticationProperties)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultSignInScheme"/>.
-        /// If only a single sign in handler scheme exists, that will be used, if more than one exists,
-        /// this will fallback to <see cref="GetDefaultAuthenticateSchemeAsync"/>.
+        /// Otherwise, this will fallback to <see cref="AuthenticationOptions.DefaultScheme"/>.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.SignInAsync(HttpContext, string, System.Security.Claims.ClaimsPrincipal, AuthenticationProperties)"/>.</returns>
         public virtual Task<AuthenticationScheme> GetDefaultSignInSchemeAsync()
-        {
-            if (_options.DefaultSignInScheme != null)
-            {
-                return GetSchemeAsync(_options.DefaultSignInScheme);
-            }
-            if (_signInHandlers.Count == 1)
-            {
-                return Task.FromResult(_signInHandlers[0]);
-            }
-            return GetDefaultAuthenticateSchemeAsync();
-        }
+            => _options.DefaultSignInScheme != null
+            ? GetSchemeAsync(_options.DefaultSignInScheme)
+            : GetDefaultSchemeAsync();
 
         /// <summary>
         /// Returns the scheme that will be used by default for <see cref="IAuthenticationService.SignOutAsync(HttpContext, string, AuthenticationProperties)"/>.
         /// This is typically specified via <see cref="AuthenticationOptions.DefaultSignOutScheme"/>.
-        /// If only a single sign out handler scheme exists, that will be used, if more than one exists,
-        /// this will fallback to <see cref="GetDefaultSignInSchemeAsync"/> if that supoorts sign out.
+        /// Otherwise this will fallback to <see cref="GetDefaultSignInSchemeAsync"/> if that supoorts sign out.
         /// </summary>
         /// <returns>The scheme that will be used by default for <see cref="IAuthenticationService.SignOutAsync(HttpContext, string, AuthenticationProperties)"/>.</returns>
         public virtual Task<AuthenticationScheme> GetDefaultSignOutSchemeAsync()
-        {
-            if (_options.DefaultSignOutScheme != null)
-            {
-                return GetSchemeAsync(_options.DefaultSignOutScheme);
-            }
-            if (_signOutHandlers.Count == 1)
-            {
-                return Task.FromResult(_signOutHandlers[0]);
-            }
-            return GetDefaultSignInSchemeAsync();
-        }
+            => _options.DefaultSignOutScheme != null
+            ? GetSchemeAsync(_options.DefaultSignOutScheme)
+            : GetDefaultSignInSchemeAsync();
 
         /// <summary>
         /// Returns the <see cref="AuthenticationScheme"/> matching the name, or null.
@@ -133,22 +101,14 @@ public virtual Task<AuthenticationScheme> GetDefaultSignOutSchemeAsync()
         /// <param name="name">The name of the authenticationScheme.</param>
         /// <returns>The scheme or null if not found.</returns>
         public virtual Task<AuthenticationScheme> GetSchemeAsync(string name)
-        {
-            if (_map.ContainsKey(name))
-            {
-                return Task.FromResult(_map[name]);
-            }
-            return Task.FromResult<AuthenticationScheme>(null);
-        }
+            => Task.FromResult(_map.ContainsKey(name) ? _map[name] : null);
 
         /// <summary>
         /// Returns the schemes in priority order for request handling.
         /// </summary>
         /// <returns>The schemes in priority order for request handling</returns>
         public virtual Task<IEnumerable<AuthenticationScheme>> GetRequestHandlerSchemesAsync()
-        {
-            return Task.FromResult<IEnumerable<AuthenticationScheme>>(_requestHandlers);
-        }
+            => Task.FromResult<IEnumerable<AuthenticationScheme>>(_requestHandlers);
 
         /// <summary>
         /// Registers a scheme for use by <see cref="IAuthenticationService"/>. 
@@ -170,14 +130,6 @@ public virtual void AddScheme(AuthenticationScheme scheme)
                 {
                     _requestHandlers.Add(scheme);
                 }
-                if (typeof(IAuthenticationSignInHandler).IsAssignableFrom(scheme.HandlerType))
-                {
-                    _signInHandlers.Add(scheme);
-                }
-                if (typeof(IAuthenticationSignOutHandler).IsAssignableFrom(scheme.HandlerType))
-                {
-                    _signOutHandlers.Add(scheme);
-                }
                 _map[scheme.Name] = scheme;
             }
         }
@@ -198,8 +150,6 @@ public virtual void RemoveScheme(string name)
                 {
                     var scheme = _map[name];
                     _requestHandlers.Remove(scheme);
-                    _signInHandlers.Remove(scheme);
-                    _signOutHandlers.Remove(scheme);
                     _map.Remove(name);
                 }
             }
diff --git a/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationSchemeProviderTests.cs b/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationSchemeProviderTests.cs
index fc647138..4fa0ea87 100644
--- a/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationSchemeProviderTests.cs
+++ b/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationSchemeProviderTests.cs
@@ -14,69 +14,69 @@ namespace Microsoft.AspNetCore.Authentication
     public class AuthenticationSchemeProviderTests
     {
         [Fact]
-        public async Task DefaultSignOutFallsbackToSignIn()
+        public async Task NoDefaultsByDefault()
         {
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
-                o.AddScheme<SignInHandler>("signin", "whatever");
-                o.AddScheme<Handler>("foobly", "whatever");
-                o.DefaultSignInScheme = "signin";
+                o.AddScheme<SignInHandler>("B", "whatever");
             }).BuildServiceProvider();
 
             var provider = services.GetRequiredService<IAuthenticationSchemeProvider>();
-            var scheme = await provider.GetDefaultSignOutSchemeAsync();
-            Assert.NotNull(scheme);
-            Assert.Equal("signin", scheme.Name);
+            Assert.Null(await provider.GetDefaultForbidSchemeAsync());
+            Assert.Null(await provider.GetDefaultAuthenticateSchemeAsync());
+            Assert.Null(await provider.GetDefaultChallengeSchemeAsync());
+            Assert.Null(await provider.GetDefaultSignInSchemeAsync());
+            Assert.Null(await provider.GetDefaultSignOutSchemeAsync());
         }
 
         [Fact]
-        public async Task DefaultForbidFallsbackToChallenge()
+        public async Task DefaultSchemesFallbackToDefaultScheme()
         {
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
-                o.AddScheme<Handler>("challenge", "whatever");
-                o.AddScheme<Handler>("foobly", "whatever");
-                o.DefaultChallengeScheme = "challenge";
+                o.DefaultScheme = "B";
+                o.AddScheme<SignInHandler>("B", "whatever");
             }).BuildServiceProvider();
 
             var provider = services.GetRequiredService<IAuthenticationSchemeProvider>();
-            var scheme = await provider.GetDefaultForbidSchemeAsync();
-            Assert.NotNull(scheme);
-            Assert.Equal("challenge", scheme.Name);
+            Assert.Equal("B", (await provider.GetDefaultForbidSchemeAsync()).Name);
+            Assert.Equal("B", (await provider.GetDefaultAuthenticateSchemeAsync()).Name);
+            Assert.Equal("B", (await provider.GetDefaultChallengeSchemeAsync()).Name);
+            Assert.Equal("B", (await provider.GetDefaultSignInSchemeAsync()).Name);
+            Assert.Equal("B", (await provider.GetDefaultSignOutSchemeAsync()).Name);
         }
 
+
         [Fact]
-        public async Task DefaultSchemesFallbackToOnlyScheme()
+        public async Task DefaultSignOutFallsbackToSignIn()
         {
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
-                o.AddScheme<SignInHandler>("single", "whatever");
+                o.AddScheme<SignInHandler>("signin", "whatever");
+                o.AddScheme<Handler>("foobly", "whatever");
+                o.DefaultSignInScheme = "signin";
             }).BuildServiceProvider();
 
             var provider = services.GetRequiredService<IAuthenticationSchemeProvider>();
-            Assert.Equal("single", (await provider.GetDefaultForbidSchemeAsync()).Name);
-            Assert.Equal("single", (await provider.GetDefaultAuthenticateSchemeAsync()).Name);
-            Assert.Equal("single", (await provider.GetDefaultChallengeSchemeAsync()).Name);
-            Assert.Equal("single", (await provider.GetDefaultSignInSchemeAsync()).Name);
-            Assert.Equal("single", (await provider.GetDefaultSignOutSchemeAsync()).Name);
+            var scheme = await provider.GetDefaultSignOutSchemeAsync();
+            Assert.NotNull(scheme);
+            Assert.Equal("signin", scheme.Name);
         }
 
         [Fact]
-        public async Task DefaultSchemesFallbackToAuthenticateScheme()
+        public async Task DefaultForbidFallsbackToChallenge()
         {
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
-                o.DefaultAuthenticateScheme = "B";
-                o.AddScheme<Handler>("A", "whatever");
-                o.AddScheme<SignInHandler>("B", "whatever");
+                o.AddScheme<Handler>("challenge", "whatever");
+                o.AddScheme<Handler>("foobly", "whatever");
+                o.DefaultChallengeScheme = "challenge";
             }).BuildServiceProvider();
 
             var provider = services.GetRequiredService<IAuthenticationSchemeProvider>();
-            Assert.Equal("B", (await provider.GetDefaultForbidSchemeAsync()).Name);
-            Assert.Equal("B", (await provider.GetDefaultAuthenticateSchemeAsync()).Name);
-            Assert.Equal("B", (await provider.GetDefaultChallengeSchemeAsync()).Name);
-            Assert.Equal("B", (await provider.GetDefaultSignInSchemeAsync()).Name);
-            Assert.Equal("B", (await provider.GetDefaultSignOutSchemeAsync()).Name);
+            var scheme = await provider.GetDefaultForbidSchemeAsync();
+            Assert.NotNull(scheme);
+            Assert.Equal("challenge", scheme.Name);
         }
 
         [Fact]
@@ -87,6 +87,8 @@ public async Task DefaultSchemesAreSet()
                 o.AddScheme<SignInHandler>("A", "whatever");
                 o.AddScheme<SignInHandler>("B", "whatever");
                 o.AddScheme<SignInHandler>("C", "whatever");
+                o.AddScheme<SignInHandler>("Def", "whatever");
+                o.DefaultScheme = "Def";
                 o.DefaultChallengeScheme = "A";
                 o.DefaultForbidScheme = "B";
                 o.DefaultSignInScheme = "C";
@@ -102,25 +104,6 @@ public async Task DefaultSchemesAreSet()
             Assert.Equal("A", (await provider.GetDefaultSignOutSchemeAsync()).Name);
         }
 
-        [Fact]
-        public async Task SignInSignOutDefaultsToOnlyOne()
-        {
-            var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
-            {
-                o.AddScheme<Handler>("basic", "whatever");
-                o.AddScheme<SignOutHandler>("signout", "whatever");
-                o.AddScheme<SignInHandler>("signin", "whatever");
-                o.DefaultAuthenticateScheme = "basic";
-            }).BuildServiceProvider();
-
-            var provider = services.GetRequiredService<IAuthenticationSchemeProvider>();
-            Assert.Equal("basic", (await provider.GetDefaultForbidSchemeAsync()).Name);
-            Assert.Equal("basic", (await provider.GetDefaultAuthenticateSchemeAsync()).Name);
-            Assert.Equal("basic", (await provider.GetDefaultChallengeSchemeAsync()).Name);
-            Assert.Equal("signin", (await provider.GetDefaultSignInSchemeAsync()).Name);
-            Assert.Equal("signin", (await provider.GetDefaultSignOutSchemeAsync()).Name); // Defaults to single sign in scheme
-        }
-
         [Fact]
         public async Task SignOutWillDefaultsToSignInThatDoesNotSignOut()
         {
diff --git a/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationServiceTests.cs b/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationServiceTests.cs
index 471053e6..c9fe57d9 100644
--- a/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationServiceTests.cs
+++ b/test/Microsoft.AspNetCore.Authentication.Core.Test/AuthenticationServiceTests.cs
@@ -56,6 +56,7 @@ public async Task ServicesWithDefaultIAuthenticationHandlerMethodsTest()
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
                 o.AddScheme<BaseHandler>("base", "whatever");
+                o.DefaultScheme = "base";
             }).BuildServiceProvider();
             var context = new DefaultHttpContext();
             context.RequestServices = services;
@@ -73,6 +74,7 @@ public async Task ServicesWithDefaultUberMethodsTest()
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
                 o.AddScheme<UberHandler>("base", "whatever");
+                o.DefaultScheme = "base";
             }).BuildServiceProvider();
             var context = new DefaultHttpContext();
             context.RequestServices = services;
@@ -90,6 +92,7 @@ public async Task ServicesWithDefaultSignInMethodsTest()
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
                 o.AddScheme<SignInHandler>("base", "whatever");
+                o.DefaultScheme = "base";
             }).BuildServiceProvider();
             var context = new DefaultHttpContext();
             context.RequestServices = services;
@@ -107,6 +110,7 @@ public async Task ServicesWithDefaultSignOutMethodsTest()
             var services = new ServiceCollection().AddOptions().AddAuthenticationCore(o =>
             {
                 o.AddScheme<SignOutHandler>("base", "whatever");
+                o.DefaultScheme = "base";
             }).BuildServiceProvider();
             var context = new DefaultHttpContext();
             context.RequestServices = services;
diff --git a/test/Microsoft.AspNetCore.Authentication.Core.Test/TokenExtensionTests.cs b/test/Microsoft.AspNetCore.Authentication.Core.Test/TokenExtensionTests.cs
index d8f25f95..21b78e52 100644
--- a/test/Microsoft.AspNetCore.Authentication.Core.Test/TokenExtensionTests.cs
+++ b/test/Microsoft.AspNetCore.Authentication.Core.Test/TokenExtensionTests.cs
@@ -129,7 +129,11 @@ public async Task GetTokenWorksWithDefaultAuthenticateScheme()
         {
             var context = new DefaultHttpContext();
             var services = new ServiceCollection().AddOptions()
-                .AddAuthenticationCore(o => o.AddScheme("simple", s => s.HandlerType = typeof(SimpleAuth)));
+                .AddAuthenticationCore(o =>
+                {
+                    o.DefaultScheme = "simple";
+                    o.AddScheme("simple", s => s.HandlerType = typeof(SimpleAuth));
+                });
             context.RequestServices = services.BuildServiceProvider();
 
             Assert.Equal("1", await context.GetTokenAsync("One"));