diff --git a/proxy/exchange.go b/proxy/exchange.go
index fa8b220a4..a91cb6783 100644
--- a/proxy/exchange.go
+++ b/proxy/exchange.go
@@ -97,20 +97,20 @@ func exchange(u upstream.Upstream, req *dns.Msg, c Clock) (resp *dns.Msg, dur ti
 
 // upstreamRTTStats is the statistics for a single upstream's round-trip time.
 type upstreamRTTStats struct {
-	// avgRTT is the current average round-trip time in seconds.  The float64 is
-	// the returning type of [time.Duration.Seconds] method and is used to avoid
-	// unnecessary divisions.
-	avgRTT float64
-
-	// reqNum is the number of requests to the upstream.
-	reqNum uint64
+	// rttSum is the sum of all the round-trip times in microseconds.  The
+	// float64 type is used since it's capable of representing about 285 years
+	// in microseconds.
+	rttSum float64
+
+	// reqNum is the number of requests to the upstream.  The float64 type is
+	// used since to avoid unnecessary type conversions.
+	reqNum float64
 }
 
 // update returns updated stats after adding given RTT.
 func (stats upstreamRTTStats) update(rtt time.Duration) (updated upstreamRTTStats) {
 	return upstreamRTTStats{
-		// See https://en.wikipedia.org/wiki/Moving_average#Cumulative_average.
-		avgRTT: (rtt.Seconds() + stats.avgRTT*float64(stats.reqNum)) / float64(stats.reqNum+1),
+		rttSum: stats.rttSum + float64(rtt.Microseconds()),
 		reqNum: stats.reqNum + 1,
 	}
 }
@@ -125,11 +125,11 @@ func (p *Proxy) calcWeights(ups []upstream.Upstream) (weights []float64) {
 
 	for _, u := range ups {
 		stat := p.upstreamRTTStats[u.Address()]
-		if stat.avgRTT == 0 {
+		if stat.rttSum == 0 || stat.reqNum == 0 {
 			// Use 1 as the default weight.
 			weights = append(weights, 1)
 		} else {
-			weights = append(weights, 1/stat.avgRTT)
+			weights = append(weights, 1/(stat.rttSum/stat.reqNum))
 		}
 	}
 
diff --git a/proxy/exchange_internal_test.go b/proxy/exchange_internal_test.go
index b7037276f..7fe5587d1 100644
--- a/proxy/exchange_internal_test.go
+++ b/proxy/exchange_internal_test.go
@@ -153,18 +153,18 @@ func TestProxy_Exchange_loadBalance(t *testing.T) {
 		servers  []upstream.Upstream
 	}{{
 		wantStat: map[upstream.Upstream]int64{
-			fastUps:    8906,
-			slowerUps:  920,
-			slowestUps: 174,
+			fastUps:    8917,
+			slowerUps:  911,
+			slowestUps: 172,
 		},
 		clock:   zeroingClock,
 		name:    "all_good",
 		servers: []upstream.Upstream{slowestUps, slowerUps, fastUps},
 	}, {
 		wantStat: map[upstream.Upstream]int64{
-			fastUps:   9074,
-			slowerUps: 926,
-			err1Ups:   8,
+			fastUps:   9081,
+			slowerUps: 919,
+			err1Ups:   7,
 		},
 		clock:   zeroingClock,
 		name:    "one_bad",
@@ -179,8 +179,8 @@ func TestProxy_Exchange_loadBalance(t *testing.T) {
 		servers: []upstream.Upstream{err2Ups, err1Ups},
 	}, {
 		wantStat: map[upstream.Upstream]int64{
-			fastUps:    7806,
-			slowerUps:  830,
+			fastUps:    7803,
+			slowerUps:  833,
 			fastestUps: 1365,
 		},
 		clock:   zeroingClock,
@@ -188,9 +188,9 @@ func TestProxy_Exchange_loadBalance(t *testing.T) {
 		servers: []upstream.Upstream{fastUps, slowerUps, fastestUps},
 	}, {
 		wantStat: map[upstream.Upstream]int64{
-			each200: 5308,
-			each100: 3099,
-			each50:  1682,
+			each200: 5316,
+			each100: 3090,
+			each50:  1683,
 		},
 		clock:   constClock,
 		name:    "error_each_nth",