Merge pull request #77 from haydenroche5/cobs

Author: haydenroche5

.github/workflows/hil-circuitpython.yml

@@ -21,6 +21,9 @@ on:
         type: boolean
         default: true
 
+  schedule:
+    - cron: '30 4 * * 1'
+
 jobs:
   test:
     runs-on: [self-hosted, linux, circuitpython, swan-3.0, notecard-serial]

Makefile

@@ -29,7 +29,7 @@ flake8:
 	# F403 'from module import *' used; unable to detect undefined names  https://www.flake8rules.com/rules/F403.html
 	# W503 Line break occurred before a binary operator https://www.flake8rules.com/rules/W503.html
 	# E501 Line too long (>79 characters) https://www.flake8rules.com/rules/E501.html
-	${PYTHON} -m flake8 test/ notecard/ examples/ mpy_board/ --count --ignore=E722,F401,F403,W503,E501,E502 --show-source --statistics
+	${PYTHON} -m flake8 --exclude=notecard/md5.py test/ notecard/ examples/ mpy_board/ --count --ignore=E722,F401,F403,W503,E501,E502 --show-source --statistics
 
 coverage:
 	${RUN_VENV_ACTIVATE}

examples/binary-mode/binary_loopback_example.py

@@ -0,0 +1,110 @@
+"""note-python binary loopback example.
+
+This example writes an array of bytes to the binary data store on a Notecard and
+reads them back. It checks that what was written exactly matches what's read
+back.
+
+Supports MicroPython, CircuitPython, and Raspberry Pi (Linux).
+"""
+import sys
+
+
+def run_example(product_uid, use_uart=True):
+    """Connect to Notcard and run a binary loopback test."""
+    tx_buf = bytearray([
+        0x67, 0x48, 0xa8, 0x1e, 0x9f, 0xbb, 0xb7, 0x27, 0xbb, 0x31, 0x89, 0x00, 0x1f,
+        0x60, 0x49, 0x8a, 0x63, 0xa1, 0x2b, 0xac, 0xb8, 0xa9, 0xb0, 0x59, 0x71, 0x65,
+        0xdd, 0x87, 0x73, 0x8a, 0x06, 0x9d, 0x40, 0xc1, 0xee, 0x24, 0xca, 0x31, 0xee,
+        0x88, 0xf7, 0xf1, 0x23, 0x60, 0xf2, 0x01, 0x98, 0x39, 0x21, 0x18, 0x25, 0x3c,
+        0x36, 0xf7, 0x93, 0xae, 0x50, 0xd6, 0x7d, 0x93, 0x55, 0xff, 0xcb, 0x56, 0xd3,
+        0xd3, 0xd5, 0xe9, 0xf0, 0x60, 0xf7, 0xe9, 0xd3, 0xa4, 0x40, 0xe7, 0x8a, 0x71,
+        0x72, 0x8b, 0x28, 0x5d, 0x57, 0x57, 0x8c, 0xc3, 0xd4, 0xe2, 0x05, 0xfa, 0x98,
+        0xd2, 0x26, 0x4f, 0x5d, 0xb3, 0x08, 0x02, 0xf2, 0x50, 0x23, 0x5d, 0x9c, 0x6e,
+        0x63, 0x7e, 0x03, 0x22, 0xa5, 0xb3, 0x5e, 0x95, 0xf2, 0x74, 0xfd, 0x3c, 0x2d,
+        0x06, 0xf8, 0xdc, 0x34, 0xe4, 0x3d, 0x42, 0x47, 0x7c, 0x61, 0xe6, 0xe1, 0x53
+    ])
+
+    biggest_notecard_response = 400
+    binary_chunk_size = 32
+    uart_rx_buf_size = biggest_notecard_response + binary_chunk_size
+
+    if sys.implementation.name == 'micropython':
+        from machine import UART
+        from machine import I2C
+        from machine import Pin
+        import board
+
+        if use_uart:
+            port = UART(board.UART, 9600)
+            port.init(9600, bits=8, parity=None, stop=1,
+                      timeout=3000, timeout_char=100, rxbuf=uart_rx_buf_size)
+        else:
+            port = I2C(board.I2C_ID, scl=Pin(board.SCL), sda=Pin(board.SDA))
+    elif sys.implementation.name == 'circuitpython':
+        import busio
+        import board
+
+        if use_uart:
+            port = busio.UART(board.TX, board.RX, baudrate=9600, receiver_buffer_size=uart_rx_buf_size)
+        else:
+            port = busio.I2C(board.SCL, board.SDA)
+    else:
+        import os
+
+        sys.path.insert(0, os.path.abspath(
+                        os.path.join(os.path.dirname(__file__), '..')))
+
+        from periphery import I2C
+        import serial
+
+        if use_uart:
+            port = serial.Serial('/dev/ttyACM0', 9600)
+        else:
+            port = I2C('/dev/i2c-1')
+
+    import notecard
+    from notecard import binary_helpers
+
+    if use_uart:
+        card = notecard.OpenSerial(port, debug=True)
+    else:
+        card = notecard.OpenI2C(port, 0, 0, debug=True)
+
+    print('Clearing out any old data...')
+    binary_helpers.binary_store_reset(card)
+
+    print('Sending buffer...')
+    binary_helpers.binary_store_transmit(card, tx_buf, 0)
+    print(f'Sent {len(tx_buf)} bytes to the Notecard.')
+
+    print('Reading it back...')
+    rx_buf = bytearray()
+
+    left = binary_helpers.binary_store_decoded_length(card)
+    offset = 0
+    while left > 0:
+        chunk_size = left if binary_chunk_size > left else binary_chunk_size
+        chunk = binary_helpers.binary_store_receive(card, offset, chunk_size)
+        rx_buf.extend(chunk)
+        left -= chunk_size
+        offset += chunk_size
+
+    print(f'Received {len(rx_buf)} bytes from the Notecard.')
+
+    print('Checking if received matches transmitted...')
+    rx_len = len(rx_buf)
+    tx_len = len(tx_buf)
+    assert rx_len == tx_len, f'Length mismatch between sent and received data. Sent {tx_len} bytes. Received {rx_len} bytes.'
+
+    for idx, (tx_byte, rx_byte) in enumerate(zip(tx_buf, rx_buf)):
+        assert tx_byte == rx_byte, f'Data mismatch detected at index {idx}. Sent: {tx_byte}. Received: {rx_byte}.'
+
+    print('Received matches transmitted.')
+    print('Example complete.')
+
+
+if __name__ == '__main__':
+    product_uid = 'com.your-company.your-project'
+    # Choose either UART or I2C for Notecard
+    use_uart = True
+    run_example(product_uid, use_uart)

notecard/binary_helpers.py

@@ -0,0 +1,145 @@
+"""Helper methods for doing binary transfers to/from a Notecard."""
+
+import sys
+from .cobs import cobs_encode, cobs_decode
+from .notecard import Notecard, CARD_INTRA_TRANSACTION_TIMEOUT_SEC
+
+BINARY_RETRIES = 2
+
+if sys.implementation.name == 'cpython':
+    import hashlib
+
+    def _md5_hash(data):
+        """Create an MD5 digest of the given data."""
+        return hashlib.md5(data).hexdigest()
+else:
+    from .md5 import digest as _md5_hash
+
+
+def binary_store_decoded_length(card: Notecard):
+    """Get the length of the decoded binary data store."""
+    rsp = card.Transaction({'req': 'card.binary'})
+    # Ignore {bad-bin} errors, but fail on other types of errors.
+    if 'err' in rsp and '{bad-bin}' not in rsp['err']:
+        raise Exception(
+            f'Error in response to card.binary request: {rsp["err"]}.')
+
+    return rsp['length'] if 'length' in rsp else 0
+
+
+def binary_store_reset(card: Notecard):
+    """Reset the binary data store."""
+    rsp = card.Transaction({'req': 'card.binary', 'delete': True})
+    if 'err' in rsp:
+        raise Exception(
+            f'Error in response to card.binary delete request: {rsp["err"]}.')
+
+
+def binary_store_transmit(card: Notecard, data: bytearray, offset: int):
+    """Write bytes to index `offset` of the binary data store."""
+    # Make a copy of the data to transmit. We do not modify the user's passed in
+    # `data` object.
+    tx_data = bytearray(data)
+    rsp = card.Transaction({'req': 'card.binary'})
+
+    # Ignore `{bad-bin}` errors, because we intend to overwrite the data.
+    if 'err' in rsp and '{bad-bin}' not in rsp['err']:
+        raise Exception(rsp['err'])
+
+    if 'max' not in rsp or rsp['max'] == 0:
+        raise Exception(('Unexpected card.binary response: max is zero or not '
+                         'present.'))
+
+    curr_len = rsp['length'] if 'length' in rsp else 0
+    if offset != curr_len:
+        raise Exception('Notecard data length is misaligned with offset.')
+
+    max_len = rsp['max']
+    remaining = max_len - curr_len if offset > 0 else max_len
+    if len(tx_data) > remaining:
+        raise Exception(('Data to transmit won\'t fit in the Notecard\'s binary'
+                         ' store.'))
+
+    encoded = cobs_encode(tx_data, ord('\n'))
+    req = {
+        'req': 'card.binary.put',
+        'cobs': len(encoded),
+        'status': _md5_hash(tx_data)
+    }
+    encoded.append(ord('\n'))
+    if offset > 0:
+        req['offset'] = offset
+
+    tries = 1 + BINARY_RETRIES
+    while tries > 0:
+        try:
+            # We need to hold the lock for both the card.binary.put transaction
+            # and the subsequent transmission of the binary data.
+            card.lock()
+
+            # Pass lock=false because we're already locked.
+            rsp = card.Transaction(req, lock=False)
+            if 'err' in rsp:
+                raise Exception(rsp['err'])
+
+            # Send the binary data.
+            card.transmit(encoded, delay=False)
+        finally:
+            card.unlock()
+
+        rsp = card.Transaction({'req': 'card.binary'})
+        if 'err' in rsp:
+            # Retry on {bad-bin} errors.
+            if '{bad-bin}' in rsp['err']:
+                tries -= 1
+
+                if card._debug and tries > 0:
+                    print('Error during binary transmission, retrying...')
+            # Fail on all other error types.
+            else:
+                raise Exception(rsp['err'])
+        else:
+            break
+
+    if tries == 0:
+        raise Exception('Failed to transmit binary data.')
+
+
+def binary_store_receive(card, offset: int, length: int):
+    """Receive `length' bytes from index `offset` of the binary data store."""
+    req = {
+        'req': 'card.binary.get',
+        'offset': offset,
+        'length': length
+    }
+    try:
+        # We need to hold the lock for both the card.binary.get transaction
+        # and the subsequent receipt of the binary data.
+        card.lock()
+
+        # Pass lock=false because we're already locked.
+        rsp = card.Transaction(req, lock=False)
+        if 'err' in rsp:
+            raise Exception(rsp['err'])
+
+        # Receive the binary data, keeping everything except the last byte,
+        # which is a newline.
+        try:
+            encoded = card.receive(delay=False)[:-1]
+        except Exception as e:
+            # Queue up a reset if there was an issue receiving the binary data.
+            # The reset will attempt to drain the binary data from the Notecard
+            # so that the comms channel with the Notecard is clean before the
+            # next transaction.
+            card._reset_required = True
+            raise e
+
+    finally:
+        card.unlock()
+
+    decoded = cobs_decode(encoded, ord('\n'))
+
+    if _md5_hash(decoded) != rsp['status']:
+        raise Exception('Computed MD5 does not match received MD5.')
+
+    return decoded

notecard/cobs.py

@@ -0,0 +1,53 @@
+"""Methods for COBS encoding and decoding arbitrary bytearrays."""
+
+
+def cobs_encode(data: bytearray, eop: int) -> bytearray:
+    """COBS encode an array of bytes, using eop as the end of packet marker."""
+    cobs_overhead = 1 + (len(data) // 254)
+    encoded = bytearray(len(data) + cobs_overhead)
+    code = 1
+    idx = 0
+    code_idx = idx
+    idx += 1
+
+    for byte in data:
+        if byte != 0:
+            encoded[idx] = byte ^ eop
+            idx += 1
+            code += 1
+        if byte == 0 or code == 0xFF:
+            encoded[code_idx] = code ^ eop
+            code = 1
+            code_idx = idx
+            idx += 1
+
+    encoded[code_idx] = code ^ eop
+
+    return encoded[:idx]
+
+
+def cobs_decode(encoded: bytes, eop: int) -> bytearray:
+    """COBS decode an array of bytes, using eop as the end of packet marker."""
+    decoded = bytearray(len(encoded))
+    idx = 0
+    copy = 0
+    code = 0xFF
+
+    for byte in encoded:
+        if copy != 0:
+            decoded[idx] = byte ^ eop
+            idx += 1
+        else:
+            if code != 0xFF:
+                decoded[idx] = 0
+                idx += 1
+
+            copy = byte ^ eop
+            code = copy
+
+            if code == 0:
+                break
+
+        copy -= 1
+
+    return decoded[:idx]