Fixing how ring ownership is calculated on the ring status page

Signed-off-by: alanprot <alanprot@gmail.com>

Author: alanprot

pkg/ring/http.go

@@ -38,6 +38,7 @@ const pageContent = `
 						<th>Last Heartbeat</th>
 						<th>Tokens</th>
 						<th>Ownership</th>
+						<th>Ownership Diff From Expected</th>
 						<th>Actions</th>
 					</tr>
 				</thead>
@@ -56,6 +57,7 @@ const pageContent = `
 						<td>{{ .HeartbeatTimestamp }}</td>
 						<td>{{ .NumTokens }}</td>
 						<td>{{ .Ownership }}%</td>
+						<td>{{ .DiffOwnership }}%</td>
 						<td><button name="forget" value="{{ .ID }}" type="submit">Forget</button></td>
 					</tr>
 					{{ end }}
@@ -114,6 +116,7 @@ type ingesterDesc struct {
 	Tokens              []uint32 `json:"tokens"`
 	NumTokens           int      `json:"-"`
 	Ownership           float64  `json:"-"`
+	DiffOwnership       float64  `json:"-"`
 }
 
 type httpResponse struct {
@@ -154,6 +157,7 @@ func (r *Ring) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	storageLastUpdate := r.KVClient.LastUpdateTime(r.key)
 	var ingesters []ingesterDesc
 	_, owned := r.countTokens()
+	numTokens, ownedByAz := r.countTokensByAz()
 	for _, id := range ingesterIDs {
 		ing := r.ringDesc.Ingesters[id]
 		heartbeatTimestamp := time.Unix(ing.Timestamp, 0)
@@ -168,6 +172,19 @@ func (r *Ring) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			registeredTimestamp = ing.GetRegisteredAt().String()
 		}
 
+		var ownership float64
+		var deltaOwnership float64
+
+		if r.cfg.ZoneAwarenessEnabled {
+			ownership = (float64(ownedByAz[id]) / float64(math.MaxUint32)) * 100
+			expectedOwnership := 1 / float64(len(numTokens[ing.Zone])) * 100
+			deltaOwnership = (1 - expectedOwnership/ownership) * 100
+		} else {
+			ownership = (float64(owned[id]) / float64(math.MaxUint32)) * 100
+			expectedOwnership := 1 / float64(len(owned)) * 100
+			deltaOwnership = (1 - expectedOwnership/ownership) * 100
+		}
+
 		ingesters = append(ingesters, ingesterDesc{
 			ID:                  id,
 			State:               state,
@@ -177,7 +194,8 @@ func (r *Ring) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			Tokens:              ing.Tokens,
 			Zone:                ing.Zone,
 			NumTokens:           len(ing.Tokens),
-			Ownership:           (float64(owned[id]) / float64(math.MaxUint32)) * 100,
+			Ownership:           ownership,
+			DiffOwnership:       deltaOwnership,
 		})
 	}
 

pkg/ring/ring.go

@@ -590,22 +590,42 @@ func (r *Ring) GetReplicationSetForOperation(op Operation) (ReplicationSet, erro
 	}, nil
 }
 
+func (r *Ring) countTokensByAz() (map[string]map[string]uint32, map[string]int64) {
+	numTokens := map[string]map[string]uint32{}
+	owned := map[string]int64{}
+
+	for zone, zonalTokens := range r.ringDesc.getTokensByZone() {
+		numTokens[zone] = map[string]uint32{}
+		for i := 1; i <= len(zonalTokens); i++ {
+			index := i % len(zonalTokens)
+			diff := tokenDistance(zonalTokens[i-1], zonalTokens[index])
+			info := r.ringInstanceByToken[zonalTokens[index]]
+			owned[info.InstanceID] = owned[info.InstanceID] + diff
+			numTokens[zone][info.InstanceID] = numTokens[zone][info.InstanceID] + 1
+		}
+	}
+
+	// Set to 0 the number of owned tokens by instances which don't have tokens yet.
+	for id, info := range r.ringDesc.Ingesters {
+		if _, ok := owned[id]; !ok {
+			owned[id] = 0
+			numTokens[info.Zone][id] = 0
+		}
+	}
+
+	return numTokens, owned
+}
+
 // countTokens returns the number of tokens and tokens within the range for each instance.
 // The ring read lock must be already taken when calling this function.
-func (r *Ring) countTokens() (map[string]uint32, map[string]uint32) {
-	owned := map[string]uint32{}
+func (r *Ring) countTokens() (map[string]uint32, map[string]int64) {
+	owned := map[string]int64{}
 	numTokens := map[string]uint32{}
-	for i, token := range r.ringTokens {
-		var diff uint32
-
-		// Compute how many tokens are within the range.
-		if i+1 == len(r.ringTokens) {
-			diff = (math.MaxUint32 - token) + r.ringTokens[0]
-		} else {
-			diff = r.ringTokens[i+1] - token
-		}
+	for i := 1; i <= len(r.ringTokens); i++ { // Compute how many tokens are within the range.
+		index := i % len(r.ringTokens)
+		diff := tokenDistance(r.ringTokens[i-1], r.ringTokens[index])
 
-		info := r.ringInstanceByToken[token]
+		info := r.ringInstanceByToken[r.ringTokens[index]]
 		numTokens[info.InstanceID] = numTokens[info.InstanceID] + 1
 		owned[info.InstanceID] = owned[info.InstanceID] + diff
 	}