15286 do_composition() needs sign awareness

Reviewed by: Patrick Mooney <pmooney@pfmooney.com>
Reviewed by: Richard Lowe <richlowe@richlowe.net>
Approved by: Joshua M. Clulow <josh@sysmgr.org>

usr/src/common/unicode/u8_textprep.c

@@ -23,7 +23,9 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
+/*
+ * Copyright 2022 MNX Cloud, Inc.
+ */
 
 
 /*
@@ -217,10 +219,10 @@ const int8_t u8_number_of_bytes[0x100] = {
 /*	80  81  82  83  84  85  86  87  88  89  8A  8B  8C  8D  8E  8F  */
 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
 
-/*  	90  91  92  93  94  95  96  97  98  99  9A  9B  9C  9D  9E  9F  */
+/*	90  91  92  93  94  95  96  97  98  99  9A  9B  9C  9D  9E  9F  */
 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
 
-/*  	A0  A1  A2  A3  A4  A5  A6  A7  A8  A9  AA  AB  AC  AD  AE  AF  */
+/*	A0  A1  A2  A3  A4  A5  A6  A7  A8  A9  AA  AB  AC  AD  AE  AF  */
 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
 
 /*	B0  B1  B2  B3  B4  B5  B6  B7  B8  B9  BA  BB  BC  BD  BE  BF  */
@@ -566,7 +568,7 @@ do_case_conv(int uv, uchar_t *u8s, uchar_t *s, int sz, boolean_t is_it_toupper)
  */
 static int
 do_case_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1,
-	size_t n2, boolean_t is_it_toupper, int *errnum)
+    size_t n2, boolean_t is_it_toupper, int *errnum)
 {
 	int f;
 	int sz1;
@@ -744,7 +746,7 @@ combining_class(size_t uv, uchar_t *s, size_t sz)
  */
 static size_t
 do_decomp(size_t uv, uchar_t *u8s, uchar_t *s, int sz,
-	boolean_t canonical_decomposition, u8_normalization_states_t *state)
+    boolean_t canonical_decomposition, u8_normalization_states_t *state)
 {
 	uint16_t b1 = 0;
 	uint16_t b2 = 0;
@@ -1056,7 +1058,7 @@ blocked(uchar_t *comb_class, size_t last)
  */
 static size_t
 do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start,
-	uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast)
+    uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast)
 {
 	uchar_t t[U8_STREAM_SAFE_TEXT_MAX + 1];
 	uchar_t tc[U8_MB_CUR_MAX];
@@ -1286,8 +1288,12 @@ do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start,
 		saved_l = l - disp[last];
 
 		while (p < oslast) {
-			size = u8_number_of_bytes[*p];
-			if (size <= 1 || (p + size) > oslast)
+			int8_t number_of_bytes = u8_number_of_bytes[*p];
+
+			if (number_of_bytes <= 1)
+				break;
+			size = number_of_bytes;
+			if ((p + size) > oslast)
 				break;
 
 			saved_p = p;
@@ -1378,12 +1384,12 @@ do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start,
  */
 static size_t
 collect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast,
-	boolean_t is_it_toupper,
-	boolean_t is_it_tolower,
-	boolean_t canonical_decomposition,
-	boolean_t compatibility_decomposition,
-	boolean_t canonical_composition,
-	int *errnum, u8_normalization_states_t *state)
+    boolean_t is_it_toupper,
+    boolean_t is_it_tolower,
+    boolean_t canonical_decomposition,
+    boolean_t compatibility_decomposition,
+    boolean_t canonical_composition,
+    int *errnum, u8_normalization_states_t *state)
 {
 	uchar_t *s;
 	int sz;
@@ -1726,7 +1732,7 @@ collect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast,
  */
 static int
 do_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2,
-	int flag, int *errnum)
+    int flag, int *errnum)
 {
 	int result;
 	size_t sz1;
@@ -1842,7 +1848,7 @@ do_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2,
  */
 int
 u8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv,
-		int *errnum)
+    int *errnum)
 {
 	int f;
 	size_t n1;
@@ -1912,7 +1918,7 @@ u8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv,
 
 size_t
 u8_textprep_str(char *inarray, size_t *inlen, char *outarray, size_t *outlen,
-	int flag, size_t unicode_version, int *errnum)
+    int flag, size_t unicode_version, int *errnum)
 {
 	int f;
 	int sz;