From afa0a230c24c2016d0fd62a15c3681e36c549219 Mon Sep 17 00:00:00 2001
From: Nebojsa Cvetkovic <nebkat@gmail.com>
Date: Wed, 2 Oct 2024 17:42:31 -0600
Subject: [PATCH] feat: Byte type 'B'

---
 bjdata/decoder.py | 63 ++++++++++++++++++++++++-----------------------
 bjdata/encoder.py | 12 ++++-----
 bjdata/markers.py |  1 +
 src/decoder.c     | 10 +++++---
 src/encoder.c     |  2 +-
 src/markers.h     |  1 +
 6 files changed, 48 insertions(+), 41 deletions(-)

diff --git a/bjdata/decoder.py b/bjdata/decoder.py
index 84a3302..3fbde6b 100644
--- a/bjdata/decoder.py
+++ b/bjdata/decoder.py
@@ -22,20 +22,21 @@
 from functools import reduce
 
 from .compat import raise_from, intern_unicode
-from .markers import (TYPE_NONE, TYPE_NULL, TYPE_NOOP, TYPE_BOOL_TRUE, TYPE_BOOL_FALSE, TYPE_INT8, TYPE_UINT8,
+from .markers import (TYPE_NONE, TYPE_NULL, TYPE_NOOP, TYPE_BOOL_TRUE, TYPE_BOOL_FALSE, TYPE_BYTE, TYPE_INT8, TYPE_UINT8,
                       TYPE_INT16, TYPE_INT32, TYPE_INT64, TYPE_FLOAT32, TYPE_FLOAT64, TYPE_HIGH_PREC, TYPE_CHAR,
 		      TYPE_UINT16, TYPE_UINT32, TYPE_UINT64, TYPE_FLOAT16,
                       TYPE_STRING, OBJECT_START, OBJECT_END, ARRAY_START, ARRAY_END, CONTAINER_TYPE, CONTAINER_COUNT)
 from numpy import array as ndarray, dtype as npdtype, frombuffer as buffer2numpy, half as halfprec
 from array import array as typedarray
 
-__TYPES = frozenset((TYPE_NULL, TYPE_BOOL_TRUE, TYPE_BOOL_FALSE, TYPE_INT8, TYPE_UINT8, TYPE_INT16, TYPE_INT32,
-                     TYPE_INT64, TYPE_FLOAT32, TYPE_FLOAT64, TYPE_UINT16, TYPE_UINT32, TYPE_UINT64, TYPE_FLOAT16, 
-		     TYPE_HIGH_PREC, TYPE_CHAR, TYPE_STRING, ARRAY_START, OBJECT_START))
+__TYPES = frozenset((TYPE_NULL, TYPE_BOOL_TRUE, TYPE_BOOL_FALSE, TYPE_BYTE, TYPE_INT8, TYPE_UINT8, TYPE_INT16,
+                     TYPE_INT32, TYPE_INT64, TYPE_FLOAT32, TYPE_FLOAT64, TYPE_UINT16, TYPE_UINT32, TYPE_UINT64,
+                     TYPE_FLOAT16, TYPE_HIGH_PREC, TYPE_CHAR, TYPE_STRING, ARRAY_START, OBJECT_START))
 __TYPES_NO_DATA = frozenset((TYPE_NULL, TYPE_BOOL_FALSE, TYPE_BOOL_TRUE))
-__TYPES_INT = frozenset((TYPE_INT8, TYPE_UINT8, TYPE_INT16, TYPE_INT32, TYPE_INT64, TYPE_UINT16, TYPE_UINT32, TYPE_UINT64))
-__TYPES_FIXLEN = frozenset((TYPE_INT8, TYPE_UINT8, TYPE_INT16, TYPE_INT32, TYPE_INT64, TYPE_UINT16, TYPE_UINT32, TYPE_UINT64,
-                     TYPE_FLOAT16, TYPE_FLOAT32, TYPE_FLOAT64, TYPE_CHAR))
+__TYPES_INT = frozenset((TYPE_BYTE, TYPE_INT8, TYPE_UINT8, TYPE_INT16, TYPE_INT32, TYPE_INT64, TYPE_UINT16,
+                         TYPE_UINT32, TYPE_UINT64))
+__TYPES_FIXLEN = frozenset((TYPE_BYTE, TYPE_INT8, TYPE_UINT8, TYPE_INT16, TYPE_INT32, TYPE_INT64, TYPE_UINT16, TYPE_UINT32,
+                            TYPE_UINT64, TYPE_FLOAT16, TYPE_FLOAT32, TYPE_FLOAT64, TYPE_CHAR))
 
 __SMALL_INTS_DECODED = [{pack('>b', i): i for i in range(-128, 128)}, {pack('<b', i): i for i in range(-128, 128)}]
 __SMALL_UINTS_DECODED = [{pack('>B', i): i for i in range(256)}, {pack('<B', i): i for i in range(256)}]
@@ -49,7 +50,8 @@
 __UNPACK_FLOAT32 = [Struct('>f').unpack, Struct('<f').unpack]
 __UNPACK_FLOAT64 = [Struct('>d').unpack, Struct('<d').unpack]
 
-__DTYPE_MAP = { TYPE_INT8: 'b',
+__DTYPE_MAP = { TYPE_BYTE: 'B',
+                TYPE_INT8: 'b',
                 TYPE_UINT8: 'B',
                 TYPE_INT16: 'h',
                 TYPE_UINT16: 'H',
@@ -62,7 +64,8 @@
                 TYPE_FLOAT64: 'd',
                 TYPE_CHAR: 'c'}
 
-__DTYPELEN_MAP={ TYPE_INT8: 1,
+__DTYPELEN_MAP={ TYPE_BYTE: 1,
+                TYPE_INT8: 1,
                 TYPE_UINT8: 1,
                 TYPE_INT16: 2,
                 TYPE_UINT16: 2,
@@ -227,6 +230,7 @@ def __decode_object_key(fp_read, marker, intern_object_keys, le=1):
 __METHOD_MAP = {TYPE_NULL: (lambda _, __, ___: None),
                 TYPE_BOOL_TRUE: (lambda _, __, ___: True),
                 TYPE_BOOL_FALSE: (lambda _, __, ___: False),
+                TYPE_BYTE: __decode_uint8,
                 TYPE_INT8: __decode_int8,
                 TYPE_UINT8: __decode_uint8,
                 TYPE_INT16: __decode_int16,
@@ -248,7 +252,7 @@ def prodlist(mylist):
          result = result * x
     return result
 
-def __get_container_params(fp_read, in_mapping, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle):
+def __get_container_params(fp_read, in_mapping, object_hook, object_pairs_hook, intern_object_keys, islittle):
     marker = fp_read(1)
     dims = []
     if marker == CONTAINER_TYPE:
@@ -262,7 +266,7 @@ def __get_container_params(fp_read, in_mapping, no_bytes, object_hook, object_pa
     if marker == CONTAINER_COUNT:
         marker = fp_read(1)
         if marker == ARRAY_START:
-            dims = __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
+            dims = __decode_array(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle)
             count = prodlist(dims)
         else:
             count = __decode_int_non_negative(fp_read, marker, islittle)
@@ -270,7 +274,7 @@ def __get_container_params(fp_read, in_mapping, no_bytes, object_hook, object_pa
 
         # special cases (no data (None or bool) / bytes array) will be handled in calling functions
         if not (type_ in __TYPES_NO_DATA or
-                (type_ == TYPE_UINT8 and not in_mapping and not no_bytes)):
+                (type_ == TYPE_BYTE and not in_mapping)):
             # Reading ahead is just to capture type, which will not exist if type is fixed
             marker = fp_read(1) if (in_mapping or type_ == TYPE_NONE) else type_
 
@@ -283,9 +287,9 @@ def __get_container_params(fp_read, in_mapping, no_bytes, object_hook, object_pa
     return marker, counting, count, type_, dims
 
 
-def __decode_object(fp_read, no_bytes, object_hook, object_pairs_hook,  # pylint: disable=too-many-branches
+def __decode_object(fp_read, object_hook, object_pairs_hook,  # pylint: disable=too-many-branches
                     intern_object_keys, islittle):
-    marker, counting, count, type_, dims = __get_container_params(fp_read, True, no_bytes,object_hook, object_pairs_hook,intern_object_keys, islittle)
+    marker, counting, count, type_, dims = __get_container_params(fp_read, True, object_hook, object_pairs_hook,intern_object_keys, islittle)
     has_pairs_hook = object_pairs_hook is not None
     obj = [] if has_pairs_hook else {}
 
@@ -323,9 +327,9 @@ def __decode_object(fp_read, no_bytes, object_hook, object_pairs_hook,  # pylint
         # handle outside above except (on KeyError) so do not have unfriendly "exception within except" backtrace
         if not handled:
             if marker == ARRAY_START:
-                value = __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
+                value = __decode_array(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle)
             elif marker == OBJECT_START:
-                value = __decode_object(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
+                value = __decode_object(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle)
             else:
                 raise DecoderException('Invalid marker within object')
 
@@ -341,15 +345,15 @@ def __decode_object(fp_read, no_bytes, object_hook, object_pairs_hook,  # pylint
     return object_pairs_hook(obj) if has_pairs_hook else object_hook(obj)
 
 
-def __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle):
-    marker, counting, count, type_, dims = __get_container_params(fp_read, False, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
+def __decode_array(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle):
+    marker, counting, count, type_, dims = __get_container_params(fp_read, False, object_hook, object_pairs_hook, intern_object_keys, islittle)
 
     # special case - no data (None or bool)
     if type_ in __TYPES_NO_DATA:
         return [__METHOD_MAP[type_](fp_read, type_, islittle)] * count
 
     # special case - bytes array
-    if type_ == TYPE_UINT8 and not no_bytes and len(dims)==0:
+    if type_ == TYPE_BYTE and len(dims)==0:
         container = fp_read(count)
         if len(container) < count:
             raise DecoderException('Container bytes array too short')
@@ -388,9 +392,9 @@ def __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_obj
         # handle outside above except (on KeyError) so do not have unfriendly "exception within except" backtrace
         if not handled:
             if marker == ARRAY_START:
-                value = __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
+                value = __decode_array(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle)
             elif marker == OBJECT_START:
-                value = __decode_object(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
+                value = __decode_object(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle)
             else:
                 raise DecoderException('Invalid marker within array')
 
@@ -411,14 +415,11 @@ def __object_hook_noop(obj):
     return obj
 
 
-def load(fp, no_bytes=False, object_hook=None, object_pairs_hook=None, intern_object_keys=False, islittle=True):
+def load(fp, object_hook=None, object_pairs_hook=None, intern_object_keys=False, islittle=True):
     """Decodes and returns BJData/UBJSON from the given file-like object
 
     Args:
         fp: read([size])-able object
-        no_bytes (bool): If set, typed UBJSON arrays (uint8) will not be
-                         converted to a bytes instance and instead treated like
-                         any other array (i.e. result in a list).
         object_hook (callable): Called with the result of any object literal
                                 decoded (instead of dict).
         object_pairs_hook (callable): Called with the result of any object
@@ -454,13 +455,13 @@ def load(fp, no_bytes=False, object_hook=None, object_pairs_hook=None, intern_ob
         |                                  | (2) unicode   |
         +----------------------------------+---------------+
         | uint8, int8, int16, int32, int64 | (3) int       |
-        |                                  | (2) int, long |
+        | byte                             | (2) int, long |
         +----------------------------------+---------------+
         | float32, float64                 | float         |
         +----------------------------------+---------------+
         | high_precision                   | Decimal       |
         +----------------------------------+---------------+
-        | array (typed, uint8)             | (3) bytes     |
+        | array (typed, byte)              | (3) bytes     |
         |                                  | (2) str       |
         +----------------------------------+---------------+
         | true                             | True          |
@@ -489,9 +490,9 @@ def load(fp, no_bytes=False, object_hook=None, object_pairs_hook=None, intern_ob
             except KeyError:
                 pass
             if marker == ARRAY_START:
-                newobj.append(__decode_array(fp_read, bool(no_bytes), object_hook, object_pairs_hook, intern_object_keys, islittle))
+                newobj.append(__decode_array(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle))
             if marker == OBJECT_START:
-                newobj.append(__decode_object(fp_read, bool(no_bytes), object_hook, object_pairs_hook, intern_object_keys, islittle))
+                newobj.append(__decode_object(fp_read, object_hook, object_pairs_hook, intern_object_keys, islittle))
             raise DecoderException('Invalid marker')
         except DecoderException as ex:
             if len(newobj)>0:
@@ -505,9 +506,9 @@ def load(fp, no_bytes=False, object_hook=None, object_pairs_hook=None, intern_ob
 
     return newobj;
 
-def loadb(chars, no_bytes=False, object_hook=None, object_pairs_hook=None, intern_object_keys=False, islittle=True):
+def loadb(chars, object_hook=None, object_pairs_hook=None, intern_object_keys=False, islittle=True):
     """Decodes and returns BJData/UBJSON from the given bytes or bytesarray object. See
        load() for available arguments."""
     with BytesIO(chars) as fp:
-        return load(fp, no_bytes=no_bytes, object_hook=object_hook, object_pairs_hook=object_pairs_hook,
+        return load(fp, object_hook=object_hook, object_pairs_hook=object_pairs_hook,
                     intern_object_keys=intern_object_keys, islittle=islittle)
diff --git a/bjdata/encoder.py b/bjdata/encoder.py
index 19c321c..926e9f0 100644
--- a/bjdata/encoder.py
+++ b/bjdata/encoder.py
@@ -22,9 +22,9 @@
 from math import isinf, isnan
 
 from .compat import Mapping, Sequence, INTEGER_TYPES, UNICODE_TYPE, TEXT_TYPES, BYTES_TYPES
-from .markers import (TYPE_NULL, TYPE_BOOL_TRUE, TYPE_BOOL_FALSE, TYPE_INT8, TYPE_UINT8, TYPE_INT16, TYPE_INT32,
-                      TYPE_INT64, TYPE_UINT16, TYPE_UINT32, TYPE_UINT64, TYPE_FLOAT16, TYPE_FLOAT32, 
-		      TYPE_FLOAT64, TYPE_HIGH_PREC, TYPE_CHAR, TYPE_STRING, OBJECT_START,
+from .markers import (TYPE_NULL, TYPE_BOOL_TRUE, TYPE_BOOL_FALSE, TYPE_BYTE, TYPE_INT8, TYPE_UINT8, TYPE_INT16,
+                      TYPE_INT32, TYPE_INT64, TYPE_UINT16, TYPE_UINT32, TYPE_UINT64, TYPE_FLOAT16, TYPE_FLOAT32,
+                      TYPE_FLOAT64, TYPE_HIGH_PREC, TYPE_CHAR, TYPE_STRING, OBJECT_START,
                       OBJECT_END, ARRAY_START, ARRAY_END, CONTAINER_TYPE, CONTAINER_COUNT)
 
 # Lookup tables for encoding small intergers, pre-initialised larger integer & float packers
@@ -57,7 +57,7 @@
 }
 
 # Prefix applicable to specialised byte array container
-__BYTES_ARRAY_PREFIX = ARRAY_START + CONTAINER_TYPE + TYPE_UINT8 + CONTAINER_COUNT
+__BYTES_ARRAY_PREFIX = ARRAY_START + CONTAINER_TYPE + TYPE_BYTE + CONTAINER_COUNT
 
 
 class EncoderException(TypeError):
@@ -342,8 +342,8 @@ def dump(obj, fp, container_count=False, sort_keys=False, no_float32=True, islit
     +------------------------------+-----------------------------------+
     | Decimal                      | high_precision                    |
     +------------------------------+-----------------------------------+
-    | (3) bytes, bytearray         | array (type, uint8)               |
-    | (2) str                      | array (type, uint8)               |
+    | (3) bytes, bytearray         | array (type, byte)                |
+    | (2) str                      | array (type, byte)                |
     +------------------------------+-----------------------------------+
     | (3) collections.abc.Mapping  | object                            |
     | (2) collections.Mapping      |                                   |
diff --git a/bjdata/markers.py b/bjdata/markers.py
index 47e6978..00f7a36 100644
--- a/bjdata/markers.py
+++ b/bjdata/markers.py
@@ -22,6 +22,7 @@
 TYPE_NOOP = b'N'
 TYPE_BOOL_TRUE = b'T'
 TYPE_BOOL_FALSE = b'F'
+TYPE_BYTE = b'B'
 TYPE_INT8 = b'i'
 TYPE_UINT8 = b'U'
 TYPE_INT16 = b'I'
diff --git a/src/decoder.c b/src/decoder.c
index cc389d3..717132a 100644
--- a/src/decoder.c
+++ b/src/decoder.c
@@ -569,6 +569,7 @@ static long long _decode_int_non_negative(_bjdata_decoder_buffer_t *buffer, char
     }
 
     switch (marker) {
+        case TYPE_BYTE:
         case TYPE_UINT8:
             BAIL_ON_NULL(int_obj = _decode_uint8(buffer));
             break;
@@ -716,7 +717,7 @@ static _container_params_t _get_container_params(_bjdata_decoder_buffer_t *buffe
             case TYPE_NULL: case TYPE_BOOL_TRUE: case TYPE_BOOL_FALSE: case TYPE_CHAR: case TYPE_STRING: case TYPE_INT8:
             case TYPE_UINT8: case TYPE_INT16: case TYPE_INT32: case TYPE_INT64: case TYPE_FLOAT32: case TYPE_FLOAT64:
 #ifdef USE__BJDATA
-            case TYPE_UINT16: case TYPE_UINT32: case TYPE_UINT64: case TYPE_FLOAT16:
+            case TYPE_UINT16: case TYPE_UINT32: case TYPE_UINT64: case TYPE_FLOAT16: case TYPE_BYTE:
 #endif
             case TYPE_HIGH_PREC: case ARRAY_START: case OBJECT_START:
                 params.type = marker;
@@ -803,7 +804,8 @@ static int _is_fixed_len_type(char type) {
     return ((TYPE_INT8 == type) || (TYPE_UINT8 == type) || (TYPE_INT16 == type)
          || (TYPE_UINT16 == type) || (TYPE_INT32 == type) || (TYPE_UINT32 == type)
          || (TYPE_INT64 == type) || (TYPE_UINT64 == type) || (TYPE_CHAR == type)
-         || (TYPE_FLOAT16 == type) || (TYPE_FLOAT32 == type) || (TYPE_FLOAT64 == type));
+         || (TYPE_FLOAT16 == type) || (TYPE_FLOAT32 == type) || (TYPE_FLOAT64 == type))
+         || (TYPE_BYTE == type);
 }
 
 // Note: Does NOT reserve a new reference
@@ -821,6 +823,7 @@ static int _get_type_info(char type, int *bytelen) {
         case TYPE_INT8:
 	    *bytelen=1;
             return PyArray_BYTE;
+        case TYPE_BYTE:
         case TYPE_UINT8:
 	    *bytelen=1;
             return PyArray_UBYTE;
@@ -881,7 +884,7 @@ static PyObject* _decode_array(_bjdata_decoder_buffer_t *buffer) {
     marker = params.marker;
     if (params.counting) {
         // special case - byte array
-        if ((TYPE_UINT8 == params.type) && !buffer->prefs.no_bytes && ndims==0) {
+        if ((TYPE_BYTE == params.type) && !buffer->prefs.no_bytes && ndims==0) {
             BAIL_ON_NULL(list = PyBytes_FromStringAndSize(NULL, params.count));
             READ_INTO_OR_BAIL(params.count, PyBytes_AS_STRING(list), "bytes array");
             return list;
@@ -1211,6 +1214,7 @@ PyObject* _bjdata_decode_value(_bjdata_decoder_buffer_t *buffer, char *given_mar
         case TYPE_INT64:
             RETURN_OR_RAISE_DECODER_EXCEPTION(_decode_int64(buffer), "int64");
 #ifdef USE__BJDATA
+        case TYPE_BYTE:
         case TYPE_UINT8:
             RETURN_OR_RAISE_DECODER_EXCEPTION(_decode_uint8(buffer), "uint8");
         case TYPE_FLOAT16:
diff --git a/src/encoder.c b/src/encoder.c
index 951e302..f7dbb0b 100644
--- a/src/encoder.c
+++ b/src/encoder.c
@@ -31,7 +31,7 @@
 
 /******************************************************************************/
 
-static char bytes_array_prefix[] = {ARRAY_START, CONTAINER_TYPE, TYPE_UINT8, CONTAINER_COUNT};
+static char bytes_array_prefix[] = {ARRAY_START, CONTAINER_TYPE, TYPE_BYTE, CONTAINER_COUNT};
 
 #define POWER_TWO(x) ((long long) 1 << (x))
 
diff --git a/src/markers.h b/src/markers.h
index dbdc4af..0a37037 100644
--- a/src/markers.h
+++ b/src/markers.h
@@ -26,6 +26,7 @@ extern "C" {
 #define TYPE_NOOP 'N'
 #define TYPE_BOOL_TRUE 'T'
 #define TYPE_BOOL_FALSE 'F'
+#define TYPE_BYTE 'B'
 #define TYPE_INT8 'i'
 #define TYPE_UINT8 'U'
 #define TYPE_INT16 'I'