lib: fix prefix_cmp() return values

OMGWTFBBQ

Signed-off-by: David Lamparter <equinox@diac24.net>

lib/compiler.h

@@ -165,6 +165,13 @@ extern "C" {
 		_min_a < _min_b ? _min_a : _min_b;                             \
 	})
 
+#define numcmp(a, b)                                                           \
+	({                                                                     \
+		typeof(a) _cmp_a = (a);                                        \
+		typeof(b) _cmp_b = (b);                                        \
+		(_cmp_a < _cmp_b) ? -1 : ((_cmp_a > _cmp_b) ? 1 : 0);          \
+	})
+
 #ifndef offsetof
 #ifdef __compiler_offsetof
 #define offsetof(TYPE, MEMBER) __compiler_offsetof(TYPE,MEMBER)

lib/prefix.c

@@ -712,10 +712,11 @@ int prefix_same(const struct prefix *p1, const struct prefix *p2)
 }
 
 /*
- * Return 0 if the network prefixes represented by the struct prefix
- * arguments are the same prefix, and 1 otherwise.  Network prefixes
- * are considered the same if the prefix lengths are equal and the
- * network parts are the same.  Host bits (which are considered masked
+ * Return -1/0/1 comparing the prefixes in a way that gives a full/linear
+ * order.
+ *
+ * Network prefixes are considered the same if the prefix lengths are equal
+ * and the network parts are the same.  Host bits (which are considered masked
  * by the prefix length) are not significant.  Thus, 10.0.0.1/8 and
  * 10.0.0.2/8 are considered equivalent by this routine.  Note that
  * this routine has the same return sense as strcmp (which is different
@@ -725,44 +726,43 @@ int prefix_cmp(const struct prefix *p1, const struct prefix *p2)
 {
 	int offset;
 	int shift;
+	int i;
 
 	/* Set both prefix's head pointer. */
 	const uint8_t *pp1;
 	const uint8_t *pp2;
 
 	if (p1->family != p2->family)
-		return 1;
+		return numcmp(p1->family, p2->family);
 	if (p1->family == AF_FLOWSPEC) {
 		pp1 = (const uint8_t *)p1->u.prefix_flowspec.ptr;
 		pp2 = (const uint8_t *)p2->u.prefix_flowspec.ptr;
 
 		if (p1->u.prefix_flowspec.prefixlen !=
 		    p2->u.prefix_flowspec.prefixlen)
-			return 1;
+			return numcmp(p1->u.prefix_flowspec.prefixlen,
+				      p2->u.prefix_flowspec.prefixlen);
 
 		offset = p1->u.prefix_flowspec.prefixlen;
 		while (offset--)
 			if (pp1[offset] != pp2[offset])
-				return 1;
+				return numcmp(pp1[offset], pp2[offset]);
 		return 0;
 	}
 	pp1 = p1->u.val;
 	pp2 = p2->u.val;
 
 	if (p1->prefixlen != p2->prefixlen)
-		return 1;
+		return numcmp(p1->prefixlen, p2->prefixlen);
 	offset = p1->prefixlen / PNBBY;
 	shift = p1->prefixlen % PNBBY;
 
-	if (shift)
-		if (maskbit[shift] & (pp1[offset] ^ pp2[offset]))
-			return 1;
-
-	while (offset--)
-		if (pp1[offset] != pp2[offset])
-			return 1;
+	i = memcmp(pp1, pp2, offset);
+	if (i)
+		return i;
 
-	return 0;
+	return numcmp(pp1[offset] & maskbit[shift],
+		      pp2[offset] & maskbit[shift]);
 }
 
 /*