diff --git a/server/cluster/coordinator_test.go b/server/cluster/coordinator_test.go
index 5eca7331a19..b8d9b134430 100644
--- a/server/cluster/coordinator_test.go
+++ b/server/cluster/coordinator_test.go
@@ -1044,14 +1044,25 @@ func (s *testOperatorControllerSuite) TestDownStoreLimit(c *C) {
 	region := tc.GetRegion(1)
 	tc.setStoreDown(1)
 	tc.SetStoreLimit(1, storelimit.RemovePeer, 1)
+
 	region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{
 		{
 			Peer:        region.GetStorePeer(1),
 			DownSeconds: 24 * 60 * 60,
 		},
-	}))
+	}), core.SetApproximateSize(1))
+	tc.putRegion(region)
+	for i := uint64(1); i < 20; i++ {
+		tc.addRegionStore(i+3, 100)
+		op := rc.Check(region)
+		c.Assert(op, NotNil)
+		c.Assert(oc.AddOperator(op), IsTrue)
+		oc.RemoveOperator(op)
+	}
 
-	for i := uint64(1); i <= 5; i++ {
+	region = region.Clone(core.SetApproximateSize(100))
+	tc.putRegion(region)
+	for i := uint64(20); i < 25; i++ {
 		tc.addRegionStore(i+3, 100)
 		op := rc.Check(region)
 		c.Assert(op, NotNil)
diff --git a/server/schedule/operator/influence.go b/server/schedule/operator/influence.go
index 91248d1e9d9..c4e7c58c321 100644
--- a/server/schedule/operator/influence.go
+++ b/server/schedule/operator/influence.go
@@ -81,7 +81,7 @@ func (s *StoreInfluence) addStepCost(limitType storelimit.Type, cost int64) {
 func (s *StoreInfluence) AdjustStepCost(limitType storelimit.Type, regionSize int64) {
 	if regionSize > storelimit.SmallRegionThreshold {
 		s.addStepCost(limitType, storelimit.RegionInfluence[limitType])
-	} else if regionSize <= storelimit.SmallRegionThreshold && regionSize > core.EmptyRegionApproximateSize {
+	} else if regionSize > core.EmptyRegionApproximateSize {
 		s.addStepCost(limitType, storelimit.SmallRegionInfluence[limitType])
 	}
 }
diff --git a/server/schedule/operator/step.go b/server/schedule/operator/step.go
index bb8a4743950..42240296480 100644
--- a/server/schedule/operator/step.go
+++ b/server/schedule/operator/step.go
@@ -269,9 +269,9 @@ func (rp RemovePeer) Influence(opInfluence OpInfluence, region *core.RegionInfo)
 	regionSize := region.GetApproximateSize()
 	from.RegionSize -= regionSize
 	from.RegionCount--
-	if rp.IsDownStore {
-		from.AdjustStepCost(storelimit.RemovePeer, storelimit.SmallRegionThreshold)
-		return
+
+	if rp.IsDownStore && regionSize > storelimit.SmallRegionThreshold {
+		regionSize = storelimit.SmallRegionThreshold
 	}
 	from.AdjustStepCost(storelimit.RemovePeer, regionSize)
 }