From 38b6c920abc0fb7a0c1fbedc6b851ca0109a7c56 Mon Sep 17 00:00:00 2001 From: Jake Date: Fri, 26 Mar 2021 21:45:10 +0100 Subject: [PATCH 01/35] Replace bluepy with bleak closes #5 --- requirements.txt | 2 +- setup.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e842205..485bc30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -git+https://github.com/edwios/bluepy.git \ No newline at end of file +bleak >= 0.11.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 15e54ea..1c7e831 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,9 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', + install_requires=[ + 'bleak >= 0.11.0' + ], entry_points={ 'console_scripts': [ 'ble-scan=ble_serial.scan:main', From d5294aee450a9497af2e4829a39372102bddabad Mon Sep 17 00:00:00 2001 From: Jake Date: Fri, 26 Mar 2021 21:45:41 +0100 Subject: [PATCH 02/35] Update ble-scan to use bleak --- ble_serial/scan.py | 92 +++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/ble_serial/scan.py b/ble_serial/scan.py index 25b566d..9ce72cb 100644 --- a/ble_serial/scan.py +++ b/ble_serial/scan.py @@ -1,55 +1,57 @@ -from bluepy.btle import Scanner, DefaultDelegate, ScanEntry, Peripheral, BTLEException -import argparse - -class ScanDelegate(DefaultDelegate): - def __init__(self): - DefaultDelegate.__init__(self) - - def handleDiscovery(self, dev, isNewDev, isNewData): - if isNewDev: - print(f"Discovered device: {dev.addr} -> {dev.getValueText(0x9)}") - elif isNewData: - print("Received new data from", dev.addr) - -def scan(time: float, deep: bool): - scanner = Scanner().withDelegate(ScanDelegate()) - devices = scanner.scan(time) - print(f'Found {len(devices)} devices!\n') - - for dev in devices: - print(f"Device {dev.addr} ({dev.addrType}), RSSI={dev.rssi} dB") - for (adtype, desc, value) in dev.getScanData(): - print(f" {adtype:02x}: {desc} = {value}") - if deep: - specific_scan(dev) - print() - -def specific_scan(addr: str): - try: - dev = Peripheral(deviceAddr=addr) - print_dev(dev) - dev.disconnect() - except BTLEException as e: - print(f'Could not read device: {e}') - -def print_dev(dev): - serv = dev.getServices() - for service in serv: - print(' Service:', service.uuid) - for char in service.getCharacteristics(): - print(' Characteristic:', char.uuid, char.propertiesToString()) +from bleak import BleakScanner, BleakClient +from bleak.backends.service import BleakGATTServiceCollection +import argparse, asyncio + + +async def scan(args): + if args.addr: + await deep_scan(args.addr) + else: + await general_scan(args.sec) + +async def general_scan(time: float): + print("Started BLE scan\n") + + devices = await BleakScanner.discover(timeout=time) + + sorted_devices = sorted(devices, key=lambda dev: dev.rssi, reverse=True) + for d in sorted_devices: + print(f'{d.address} (RSSI={d.rssi}): {d.name}') + + print("\nFinished BLE scan") + + +async def deep_scan(dev: str): + print(f"Started deep scan of {dev}\n") + + async with BleakClient(dev) as client: + print_details(await client.get_services()) + + print(f"\nCompleted deep scan of {dev}") + +def print_details(serv: BleakGATTServiceCollection): + INDENT = ' ' + for s in serv: + print('SERVICE', s) + for char in s.characteristics: + print(INDENT, 'CHARACTERISTIC', char, char.properties) + for desc in char.descriptors: + print(INDENT*2, 'DESCRIPTOR', desc) + def main(): parser = argparse.ArgumentParser( - description='Scanner for BLE devices and service/characteristcs. ROOT required.', + description='Scanner for BLE devices and service/characteristics.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-t', '--scan-time', dest='sec', default=5.0, type=float, help='Duration of the scan in seconds') - parser.add_argument('-d', '--deep-scan', dest='deep', action='store_true', - help='Try to connect to the devices and read out the service/characteristic UUIDs') + parser.add_argument('-d', '--deep-scan', dest='addr', type=str, + help='Try to connect to device and read out service/characteristic UUIDs') args = parser.parse_args() - scan(args.sec, args.deep) + asyncio.run(scan(args)) + if __name__ == '__main__': - main() \ No newline at end of file + # Extra function for console scripts + main() From 89a4e9814f7f08cd45ddc6fe51f498bff132267a Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 02:30:44 +0100 Subject: [PATCH 03/35] Remove bluepy submodule --- .gitmodules | 3 --- bluepy-repo | 1 - 2 files changed, 4 deletions(-) delete mode 160000 bluepy-repo diff --git a/.gitmodules b/.gitmodules index be97768..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "bluepy-repo"] - path = bluepy-repo - url = https://github.com/edwios/bluepy.git diff --git a/bluepy-repo b/bluepy-repo deleted file mode 160000 index 10f1cee..0000000 --- a/bluepy-repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 10f1cee90afb416f139949b86b491e4cfa98c886 From f753877e7993b1afd0ada7fa262d8efd513db37c Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 14:49:19 +0100 Subject: [PATCH 04/35] Replace serial thread with asyncio approach --- ble_serial/__main__.py | 9 +++-- .../linux_pty.py} | 38 +++++++++---------- 2 files changed, 25 insertions(+), 22 deletions(-) rename ble_serial/{virtual_serial.py => serial/linux_pty.py} (54%) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 7b5045e..baf652b 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,5 +1,5 @@ -import logging, sys, argparse, time -from ble_serial.virtual_serial import UART +import logging, sys, argparse, time, asyncio +from ble_serial.serial.linux_pty import UART from ble_serial.interface import BLE_interface from ble_serial.fs_log import FS_log, Direction from bluepy.btle import BTLEDisconnectError @@ -34,8 +34,10 @@ def main(): level=logging.DEBUG if args.verbose else logging.INFO ) + loop = asyncio.get_event_loop() + try: - uart = UART(args.port) + uart = UART(args.port, loop) bt = BLE_interface(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) if args.filename: log = FS_log(args.filename, args.binlog) @@ -46,6 +48,7 @@ def main(): uart.set_receiver(bt.send) logging.info('Running main loop!') uart.start() + # loop.run_forever() while True: bt.receive_loop() except BTLEDisconnectError as e: diff --git a/ble_serial/virtual_serial.py b/ble_serial/serial/linux_pty.py similarity index 54% rename from ble_serial/virtual_serial.py rename to ble_serial/serial/linux_pty.py index 291c893..7617dae 100644 --- a/ble_serial/virtual_serial.py +++ b/ble_serial/serial/linux_pty.py @@ -1,16 +1,15 @@ -import os, pty, tty, termios, logging -from select import select -from threading import Thread +import asyncio, logging +import os, pty, tty, termios -class UART(Thread): - def __init__(self, symlink: str): - super(UART, self).__init__() - self.running = True +class UART(): + def __init__(self, symlink: str, ev_loop: asyncio.AbstractEventLoop): + self.loop = ev_loop master, slave = pty.openpty() tty.setraw(master, termios.TCSANOW) self._master = master self.endpoint = os.ttyname(slave) + self.symlink = symlink os.symlink(self.endpoint, self.symlink) logging.info(f'Slave created on {self.symlink} -> {self.endpoint}') @@ -18,29 +17,30 @@ def __init__(self, symlink: str): def set_receiver(self, callback): self._cb = callback - def run(self): + + def start(self): assert self._cb, 'Receiver must be set before start!' - while self.running: - r, _, _ = select([self._master], [], [], 1.0) - if self._master in r: - data = self.read_sync() - self._cb(data) + logging.info('Starting UART event loop') + # Register the file descriptor for read event + self.loop.add_reader(self._master, self.read_handler) def stop(self): - logging.info('Stopping UART thread') - self.running = False - while self.is_alive(): - pass + logging.info('Stopping UART event loop') + # Unregister the fd + self.loop.remove_reader(self._master) os.remove(self.symlink) + def read_handler(): + data = self.read_sync() + self._cb(data) + def read_sync(self): - value = os.read(self._master, 1024) + value = os.read(self._master, 255) logging.debug(f'Read: {value}') return value def write_sync(self, value: bytes): os.write(self._master, value) logging.debug(f'Write: {value}') - From 72620d6b691d0739a41db7ccfce59d3c0601ba5b Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 16:14:05 +0100 Subject: [PATCH 05/35] Fix uart read handler --- ble_serial/serial/linux_pty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index 7617dae..5dabd95 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -32,7 +32,7 @@ def stop(self): os.remove(self.symlink) - def read_handler(): + def read_handler(self): data = self.read_sync() self._cb(data) From 5cc0c8e6c8d6f169668e20b2343c808b24f015b6 Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 18:53:51 +0100 Subject: [PATCH 06/35] Use bleak for ble_interface --- ble_serial/__main__.py | 26 ++++++++------ ble_serial/ble_interface.py | 59 ++++++++++++++++++++++++++++++ ble_serial/interface.py | 72 ------------------------------------- 3 files changed, 75 insertions(+), 82 deletions(-) create mode 100644 ble_serial/ble_interface.py delete mode 100644 ble_serial/interface.py diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index baf652b..e834989 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,8 +1,8 @@ import logging, sys, argparse, time, asyncio from ble_serial.serial.linux_pty import UART -from ble_serial.interface import BLE_interface +from ble_serial.ble_interface import BLE_interface from ble_serial.fs_log import FS_log, Direction -from bluepy.btle import BTLEDisconnectError +from bleak.exc import BleakError def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -34,11 +34,14 @@ def main(): level=logging.DEBUG if args.verbose else logging.INFO ) - loop = asyncio.get_event_loop() + asyncio.run(run(args)) + +async def run(args): + loop = asyncio.get_event_loop() try: uart = UART(args.port, loop) - bt = BLE_interface(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) + bt = BLE_interface() if args.filename: log = FS_log(args.filename, args.binlog) bt.set_receiver(log.middleware(Direction.BLE_IN, uart.write_sync)) @@ -46,13 +49,16 @@ def main(): else: bt.set_receiver(uart.write_sync) uart.set_receiver(bt.send) - logging.info('Running main loop!') + uart.start() + await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) + logging.info('Running main loop!') # loop.run_forever() while True: - bt.receive_loop() - except BTLEDisconnectError as e: - logging.warning(f'Bluetooth connection failed') + await asyncio.sleep(1) + + except BleakError as e: + logging.warning(f'Bluetooth connection failed: {e}') except KeyboardInterrupt: logging.info('Keyboard interrupt received') except Exception as e: @@ -62,11 +68,11 @@ def main(): if 'uart' in locals(): uart.stop() if 'bt' in locals(): - bt.shutdown() + await bt.shutdown() if 'log' in locals(): log.finish() logging.info('Shutdown complete.') - exit(0) + if __name__ == '__main__': main() \ No newline at end of file diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py new file mode 100644 index 0000000..5149d73 --- /dev/null +++ b/ble_serial/ble_interface.py @@ -0,0 +1,59 @@ +from bleak import BleakClient +from bleak.backends.characteristic import BleakGATTCharacteristic +from ble_serial.constants import ble_chars +import logging, asyncio +from typing import Optional + +class BLE_interface(): + async def start(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): + self.dev = BleakClient(addr_str) # addr_type only on Windows?, adapter in kwargs + await self.dev.connect() + logging.info(f'Connected device {self.dev}') + + self.write_char = self.find_char(write_uuid, 'write-without-response') + self.read_char = self.find_char(read_uuid, 'notify') + + await self.dev.start_notify(self.read_char, self.handleNotification) + + def find_char(self, uuid: Optional[str], req_prop: str) -> BleakGATTCharacteristic: + found_char = None + + # Use user supplied UUID first, otherwise try included list + if uuid: + uuid_candidates = uuid + else: + uuid_candidates = ble_chars + logging.debug(f'No {req_prop} uuid specified, trying {ble_chars}') + + for srv in self.dev.services: + for c in srv.characteristics: + if c.uuid in uuid_candidates: + found_char = c + logging.debug(f'Found {req_prop} characteristic {c}') + break + + # Check if it has the required properties + assert found_char, \ + "No characteristic with specified UUID found!" + assert (req_prop in found_char.properties), \ + f"Specified characteristic has no {req_prop} property!" + + return found_char + + def set_receiver(self, callback): + self._cb = callback + logging.info('Receiver set up') + + async def shutdown(self): + await self.dev.stop_notify(self.read_char) + await self.dev.disconnect() + logging.info('BT disconnected') + + def send(self, data: bytes): + logging.debug(f'Sending {data}') + # TODO use queue instead + asyncio.create_task(self.dev.write_gatt_char(self.write_char, data)) + + def handleNotification(self, handle: int, data: bytes): + logging.debug(f'Received notify from {handle}: {data}') + self._cb(data) \ No newline at end of file diff --git a/ble_serial/interface.py b/ble_serial/interface.py deleted file mode 100644 index 70ae2a6..0000000 --- a/ble_serial/interface.py +++ /dev/null @@ -1,72 +0,0 @@ -from bluepy.btle import Peripheral, Characteristic, Service, UUID -from bluepy.btle import DefaultDelegate, BTLEException -import logging -from ble_serial.constants import ble_chars - -class BLE_interface(): - def __init__(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): - self.dev = Peripheral(deviceAddr=addr_str, addrType=addr_type, iface=adapter) - logging.info(f'Connected device {self.dev.addr}') - - if write_uuid: - self.write_uuid = [UUID(write_uuid)] - else: - self.write_uuid = [UUID(x) for x in ble_chars] - logging.debug(f'No write uuid specified, trying {ble_chars}') - for c in self.dev.getCharacteristics(): - if c.uuid in self.write_uuid: - self._write_charac = c - self.write_uuid = self._write_charac.uuid - logging.debug(f'Found write characteristic {self.write_uuid}') - break - assert hasattr(self, '_write_charac'), \ - "No characteristic with specified UUID found!" - assert (self._write_charac.properties & Characteristic.props["WRITE_NO_RESP"]), \ - "Specified characteristic is not writable!" - - # Only subscribe to read_uuid notifications if it was specified - if read_uuid: - self.read_uuid = [UUID(read_uuid)] - for c in self.dev.getCharacteristics(): - if c.uuid in self.read_uuid: - self._read_charac = c - self.read_uuid = self._read_charac.uuid - logging.debug(f'Found read characteristic {self.read_uuid}') - break - assert hasattr(self, '_read_charac'), \ - "No read characteristic with specified UUID found!" - assert (self._read_charac.properties & Characteristic.props["NOTIFY"]), \ - "Specified read characteristic is not notifiable!" - # Attempt to subscribe to notification now (write CCCD) - # First get the Client Characteristic Configuration Descriptor - self._read_charac_cccd = self._read_charac.getDescriptors(0x2902) - assert (self._read_charac_cccd is not None), \ - "Could not find CCCD for given read UUID!" - self._read_charac_cccd = self._read_charac_cccd[0] - # Now write that we want notifications from this UUID - self._read_charac_cccd.write(bytes([0x01, 0x00])) - - def send(self, data: bytes): - logging.debug(f'Sending {data}') - self._write_charac.write(data, withResponse=False) - - def set_receiver(self, callback): - logging.info('Receiver set up') - self.dev.setDelegate(ReceiverDelegate(callback)) - - def receive_loop(self): - assert isinstance(self.dev.delegate, ReceiverDelegate), 'Callback must be set before receive loop!' - self.dev.waitForNotifications(3.0) - - def shutdown(self): - self.dev.disconnect() - logging.info('BT disconnected') - - -class ReceiverDelegate(DefaultDelegate): - def __init__(self, callback): - self._cb = callback - - def handleNotification(self, handle, data): - logging.debug(f'Received notify: {data}') - self._cb(data) \ No newline at end of file From b9a405818ffd60433690c83bc1ca3f50be249d97 Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 19:21:50 +0100 Subject: [PATCH 07/35] Queue for ble send --- ble_serial/__main__.py | 10 ++++------ ble_serial/ble_interface.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index e834989..aa13073 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -45,18 +45,16 @@ async def run(args): if args.filename: log = FS_log(args.filename, args.binlog) bt.set_receiver(log.middleware(Direction.BLE_IN, uart.write_sync)) - uart.set_receiver(log.middleware(Direction.BLE_OUT, bt.send)) + uart.set_receiver(log.middleware(Direction.BLE_OUT, bt.queue_send)) else: bt.set_receiver(uart.write_sync) - uart.set_receiver(bt.send) + uart.set_receiver(bt.queue_send) uart.start() await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) + await bt.send_loop() logging.info('Running main loop!') - # loop.run_forever() - while True: - await asyncio.sleep(1) - + except BleakError as e: logging.warning(f'Bluetooth connection failed: {e}') except KeyboardInterrupt: diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py index 5149d73..137f38d 100644 --- a/ble_serial/ble_interface.py +++ b/ble_serial/ble_interface.py @@ -13,6 +13,7 @@ async def start(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): self.write_char = self.find_char(write_uuid, 'write-without-response') self.read_char = self.find_char(read_uuid, 'notify') + self._send_queue = asyncio.Queue() await self.dev.start_notify(self.read_char, self.handleNotification) def find_char(self, uuid: Optional[str], req_prop: str) -> BleakGATTCharacteristic: @@ -44,15 +45,20 @@ def set_receiver(self, callback): self._cb = callback logging.info('Receiver set up') + async def send_loop(self): + assert hasattr(self, '_cb'), 'Callback must be set before receive loop!' + while True: + data = await self._send_queue.get() + logging.debug(f'Sending {data}') + await self.dev.write_gatt_char(self.write_char, data) + async def shutdown(self): await self.dev.stop_notify(self.read_char) await self.dev.disconnect() logging.info('BT disconnected') - def send(self, data: bytes): - logging.debug(f'Sending {data}') - # TODO use queue instead - asyncio.create_task(self.dev.write_gatt_char(self.write_char, data)) + def queue_send(self, data: bytes): + self._send_queue.put_nowait(data) def handleNotification(self, handle: int, data: bytes): logging.debug(f'Received notify from {handle}: {data}') From 80b3b774c926dc55fcb893fc1dfefcf13c2d649c Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 20:47:32 +0100 Subject: [PATCH 08/35] Slightly improve disconnect handling --- ble_serial/ble_interface.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py index 137f38d..da08127 100644 --- a/ble_serial/ble_interface.py +++ b/ble_serial/ble_interface.py @@ -1,5 +1,6 @@ from bleak import BleakClient from bleak.backends.characteristic import BleakGATTCharacteristic +from bleak.exc import BleakError from ble_serial.constants import ble_chars import logging, asyncio from typing import Optional @@ -7,6 +8,7 @@ class BLE_interface(): async def start(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): self.dev = BleakClient(addr_str) # addr_type only on Windows?, adapter in kwargs + self.dev.set_disconnected_callback(self.handle_disconnect) await self.dev.connect() logging.info(f'Connected device {self.dev}') @@ -53,8 +55,10 @@ async def send_loop(self): await self.dev.write_gatt_char(self.write_char, data) async def shutdown(self): - await self.dev.stop_notify(self.read_char) - await self.dev.disconnect() + if self.dev.is_connected: + if hasattr(self, 'read_char'): + await self.dev.stop_notify(self.read_char) + await self.dev.disconnect() logging.info('BT disconnected') def queue_send(self, data: bytes): @@ -62,4 +66,7 @@ def queue_send(self, data: bytes): def handleNotification(self, handle: int, data: bytes): logging.debug(f'Received notify from {handle}: {data}') - self._cb(data) \ No newline at end of file + self._cb(data) + + def handle_disconnect(self, client: BleakClient): + raise BleakError(f'{client.address} disconnected!') \ No newline at end of file From 61ec839d0b1c3274b6ecb14e3ee6b49646e938bd Mon Sep 17 00:00:00 2001 From: Jake Date: Sat, 27 Mar 2021 23:22:51 +0100 Subject: [PATCH 09/35] Fix bleak loglevel to INFO --- ble_serial/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index aa13073..978f4f0 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -33,6 +33,7 @@ def main(): datefmt='%H:%M:%S', level=logging.DEBUG if args.verbose else logging.INFO ) + logging.getLogger('bleak').level = logging.INFO asyncio.run(run(args)) @@ -52,8 +53,8 @@ async def run(args): uart.start() await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) - await bt.send_loop() logging.info('Running main loop!') + await bt.send_loop() except BleakError as e: logging.warning(f'Bluetooth connection failed: {e}') @@ -71,6 +72,5 @@ async def run(args): log.finish() logging.info('Shutdown complete.') - if __name__ == '__main__': main() \ No newline at end of file From e22c4c789299ef6b2e737b203c3dfa6d77e21f1d Mon Sep 17 00:00:00 2001 From: Jake Date: Sun, 28 Mar 2021 00:10:54 +0100 Subject: [PATCH 10/35] Handle uart write through queue lower packet loss in UART -> BLE --- ble_serial/__main__.py | 6 +++--- ble_serial/serial/linux_pty.py | 14 ++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 978f4f0..b12cd85 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -45,16 +45,16 @@ async def run(args): bt = BLE_interface() if args.filename: log = FS_log(args.filename, args.binlog) - bt.set_receiver(log.middleware(Direction.BLE_IN, uart.write_sync)) + bt.set_receiver(log.middleware(Direction.BLE_IN, uart.queue_write)) uart.set_receiver(log.middleware(Direction.BLE_OUT, bt.queue_send)) else: - bt.set_receiver(uart.write_sync) + bt.set_receiver(uart.queue_write) uart.set_receiver(bt.queue_send) uart.start() await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) logging.info('Running main loop!') - await bt.send_loop() + await asyncio.gather(bt.send_loop(), uart.write_loop()) except BleakError as e: logging.warning(f'Bluetooth connection failed: {e}') diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index 5dabd95..3a410ed 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -4,6 +4,7 @@ class UART(): def __init__(self, symlink: str, ev_loop: asyncio.AbstractEventLoop): self.loop = ev_loop + self._send_queue = asyncio.Queue() master, slave = pty.openpty() tty.setraw(master, termios.TCSANOW) @@ -37,10 +38,15 @@ def read_handler(self): self._cb(data) def read_sync(self): - value = os.read(self._master, 255) + value = os.read(self._master, 20) logging.debug(f'Read: {value}') return value - def write_sync(self, value: bytes): - os.write(self._master, value) - logging.debug(f'Write: {value}') + def queue_write(self, value: bytes): + self._send_queue.put_nowait(value) + + async def write_loop(self): + while True: + data = await self._send_queue.get() + logging.debug(f'Write: {data}') + os.write(self._master, data) From 158be7bfbbcda6b4aba2fd5f2760105cb5741526 Mon Sep 17 00:00:00 2001 From: Jake Date: Sun, 28 Mar 2021 00:13:48 +0100 Subject: [PATCH 11/35] Rename handleNotification to handle_notify --- ble_serial/ble_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py index da08127..99ab694 100644 --- a/ble_serial/ble_interface.py +++ b/ble_serial/ble_interface.py @@ -16,7 +16,7 @@ async def start(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): self.read_char = self.find_char(read_uuid, 'notify') self._send_queue = asyncio.Queue() - await self.dev.start_notify(self.read_char, self.handleNotification) + await self.dev.start_notify(self.read_char, self.handle_notify) def find_char(self, uuid: Optional[str], req_prop: str) -> BleakGATTCharacteristic: found_char = None @@ -64,7 +64,7 @@ async def shutdown(self): def queue_send(self, data: bytes): self._send_queue.put_nowait(data) - def handleNotification(self, handle: int, data: bytes): + def handle_notify(self, handle: int, data: bytes): logging.debug(f'Received notify from {handle}: {data}') self._cb(data) From e9fbf87889e22a30f8dfbdb16c23c266bfad2024 Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 14:42:50 +0200 Subject: [PATCH 12/35] Create main class --- ble_serial/__main__.py | 133 ++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 62 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index b12cd85..8f5a2fd 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -4,73 +4,82 @@ from ble_serial.fs_log import FS_log, Direction from bleak.exc import BleakError -def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Create virtual serial ports from BLE devices.') - - parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', - help='Increase verbosity to log all data going through') - parser.add_argument('-d', '--dev', dest='device', required=True, - help='BLE device address to connect (hex format, can be seperated by colons)') - parser.add_argument('-t', '--address-type', dest='addr_type', required=False, choices=['public', 'random'], default='public', - help='BLE address type, either public or random') - parser.add_argument('-i', '--interface', dest='adapter', required=False, default='0', - help='BLE host adapter number to use') - parser.add_argument('-w', '--write-uuid', dest='write_uuid', required=False, - help='The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out') - parser.add_argument('-l', '--log', dest='filename', required=False, - help='Enable optional logging of all bluetooth traffic to file') - parser.add_argument('-b', '--binary', dest='binlog', required=False, action='store_true', - help='Log data as raw binary, disable transformation to hex. Works only in combination with -l') - parser.add_argument('-p', '--port', dest='port', required=False, default='/tmp/ttyBLE', - help='Symlink to virtual serial port') - parser.add_argument('-r', '--read-uuid', dest='read_uuid', required=False, - help='The GATT characteristic to subscribe to notifications to read the serial data') - args = parser.parse_args() +class Main(): + def __init__(self): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description='Create virtual serial ports from BLE devices.') + + parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='Increase verbosity to log all data going through') + parser.add_argument('-d', '--dev', dest='device', required=True, + help='BLE device address to connect (hex format, can be seperated by colons)') + parser.add_argument('-t', '--address-type', dest='addr_type', required=False, choices=['public', 'random'], default='public', + help='BLE address type, either public or random') + parser.add_argument('-i', '--interface', dest='adapter', required=False, default='0', + help='BLE host adapter number to use') + parser.add_argument('-w', '--write-uuid', dest='write_uuid', required=False, + help='The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out') + parser.add_argument('-l', '--log', dest='filename', required=False, + help='Enable optional logging of all bluetooth traffic to file') + parser.add_argument('-b', '--binary', dest='binlog', required=False, action='store_true', + help='Log data as raw binary, disable transformation to hex. Works only in combination with -l') + parser.add_argument('-p', '--port', dest='port', required=False, default='/tmp/ttyBLE', + help='Symlink to virtual serial port') + parser.add_argument('-r', '--read-uuid', dest='read_uuid', required=False, + help='The GATT characteristic to subscribe to notifications to read the serial data') + self.args = parser.parse_args() - logging.basicConfig( - format='%(asctime)s.%(msecs)03d | %(levelname)s | %(filename)s: %(message)s', - datefmt='%H:%M:%S', - level=logging.DEBUG if args.verbose else logging.INFO - ) - logging.getLogger('bleak').level = logging.INFO + logging.basicConfig( + format='%(asctime)s.%(msecs)03d | %(levelname)s | %(filename)s: %(message)s', + datefmt='%H:%M:%S', + level=logging.DEBUG if self.args.verbose else logging.INFO + ) + logging.getLogger('bleak').level = logging.INFO - asyncio.run(run(args)) + asyncio.run(self.run()) + async def run(self): + args = self.args + loop = asyncio.get_event_loop() + loop.set_exception_handler(self.excp_handler) + try: + uart = UART(args.port, loop) + bt = BLE_interface() + if args.filename: + log = FS_log(args.filename, args.binlog) + bt.set_receiver(log.middleware(Direction.BLE_IN, uart.queue_write)) + uart.set_receiver(log.middleware(Direction.BLE_OUT, bt.queue_send)) + else: + bt.set_receiver(uart.queue_write) + uart.set_receiver(bt.queue_send) -async def run(args): - loop = asyncio.get_event_loop() - try: - uart = UART(args.port, loop) - bt = BLE_interface() - if args.filename: - log = FS_log(args.filename, args.binlog) - bt.set_receiver(log.middleware(Direction.BLE_IN, uart.queue_write)) - uart.set_receiver(log.middleware(Direction.BLE_OUT, bt.queue_send)) - else: - bt.set_receiver(uart.queue_write) - uart.set_receiver(bt.queue_send) + uart.start() + await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) + logging.info('Running main loop!') + self.main_loop = asyncio.gather(bt.send_loop(), uart.write_loop()) + await self.main_loop - uart.start() - await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) - logging.info('Running main loop!') - await asyncio.gather(bt.send_loop(), uart.write_loop()) + except BleakError as e: + logging.warning(f'Bluetooth connection failed: {e}') + except KeyboardInterrupt: + logging.info('Keyboard interrupt received') + except Exception as e: + logging.error(f'Unexpected Error: {e}') + finally: + logging.warning('Shutdown initiated') + if 'uart' in locals(): + uart.stop() + if 'bt' in locals(): + await bt.shutdown() + if 'log' in locals(): + log.finish() + logging.info('Shutdown complete.') - except BleakError as e: - logging.warning(f'Bluetooth connection failed: {e}') - except KeyboardInterrupt: - logging.info('Keyboard interrupt received') - except Exception as e: - logging.error(f'Unexpected Error: {e}') - finally: - logging.warning('Shutdown initiated') - if 'uart' in locals(): - uart.stop() - if 'bt' in locals(): - await bt.shutdown() - if 'log' in locals(): - log.finish() - logging.info('Shutdown complete.') + def excp_handler(self, loop: asyncio.AbstractEventLoop, context): + # Handles exception from inside bleak disconnect + # loop.default_exception_handler(context) + + print('custom handler', context['exception']) if __name__ == '__main__': - main() \ No newline at end of file + Main() \ No newline at end of file From 5051ce48c4e8a76a0a40180a7f950236a139f394 Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 15:54:13 +0200 Subject: [PATCH 13/35] Add loop stop function for interfaces + improve logging --- ble_serial/__main__.py | 40 ++++++++++++++++++---------------- ble_serial/ble_interface.py | 16 ++++++++++---- ble_serial/serial/linux_pty.py | 8 ++++++- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 8f5a2fd..c7ceee4 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -43,20 +43,20 @@ async def run(self): loop = asyncio.get_event_loop() loop.set_exception_handler(self.excp_handler) try: - uart = UART(args.port, loop) - bt = BLE_interface() + self.uart = UART(args.port, loop) + self.bt = BLE_interface() if args.filename: - log = FS_log(args.filename, args.binlog) - bt.set_receiver(log.middleware(Direction.BLE_IN, uart.queue_write)) - uart.set_receiver(log.middleware(Direction.BLE_OUT, bt.queue_send)) + self.log = FS_log(args.filename, args.binlog) + self.bt.set_receiver(self.log.middleware(Direction.BLE_IN, self.uart.queue_write)) + self.uart.set_receiver(self.log.middleware(Direction.BLE_OUT, self.bt.queue_send)) else: - bt.set_receiver(uart.queue_write) - uart.set_receiver(bt.queue_send) + self.bt.set_receiver(self.uart.queue_write) + self.uart.set_receiver(self.bt.queue_send) - uart.start() - await bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) + self.uart.start() + await self.bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) logging.info('Running main loop!') - self.main_loop = asyncio.gather(bt.send_loop(), uart.write_loop()) + self.main_loop = asyncio.gather(self.bt.send_loop(), self.uart.write_loop()) await self.main_loop except BleakError as e: @@ -67,19 +67,21 @@ async def run(self): logging.error(f'Unexpected Error: {e}') finally: logging.warning('Shutdown initiated') - if 'uart' in locals(): - uart.stop() - if 'bt' in locals(): - await bt.shutdown() - if 'log' in locals(): - log.finish() + if hasattr(self, 'uart'): + self.uart.remove() + if hasattr(self, 'bt'): + await self.bt.disconnect() + if hasattr(self, 'log'): + self.log.finish() logging.info('Shutdown complete.') + def excp_handler(self, loop: asyncio.AbstractEventLoop, context): - # Handles exception from inside bleak disconnect + # Handles exception from other tasks (inside bleak disconnect, etc) # loop.default_exception_handler(context) - - print('custom handler', context['exception']) + logging.debug(f'Asyncio execption handler called {context["exception"]}') + self.uart.stop_loop() + self.bt.stop_loop() if __name__ == '__main__': Main() \ No newline at end of file diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py index 99ab694..de5526b 100644 --- a/ble_serial/ble_interface.py +++ b/ble_serial/ble_interface.py @@ -8,14 +8,15 @@ class BLE_interface(): async def start(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): self.dev = BleakClient(addr_str) # addr_type only on Windows?, adapter in kwargs + self._send_queue = asyncio.Queue() self.dev.set_disconnected_callback(self.handle_disconnect) + logging.info(f'Trying to connect with {addr_str}') await self.dev.connect() - logging.info(f'Connected device {self.dev}') + logging.info(f'Device {self.dev.address} connected') self.write_char = self.find_char(write_uuid, 'write-without-response') self.read_char = self.find_char(read_uuid, 'notify') - self._send_queue = asyncio.Queue() await self.dev.start_notify(self.read_char, self.handle_notify) def find_char(self, uuid: Optional[str], req_prop: str) -> BleakGATTCharacteristic: @@ -51,15 +52,21 @@ async def send_loop(self): assert hasattr(self, '_cb'), 'Callback must be set before receive loop!' while True: data = await self._send_queue.get() + if data == None: + break # Let future end on shutdown logging.debug(f'Sending {data}') await self.dev.write_gatt_char(self.write_char, data) - async def shutdown(self): + def stop_loop(self): + logging.info('Stopping BLE event loop') + self._send_queue.put_nowait(None) + + async def disconnect(self): if self.dev.is_connected: if hasattr(self, 'read_char'): await self.dev.stop_notify(self.read_char) await self.dev.disconnect() - logging.info('BT disconnected') + logging.info('BT disconnected') def queue_send(self, data: bytes): self._send_queue.put_nowait(data) @@ -69,4 +76,5 @@ def handle_notify(self, handle: int, data: bytes): self._cb(data) def handle_disconnect(self, client: BleakClient): + logging.info(f'Device {client.address} disconnected') raise BleakError(f'{client.address} disconnected!') \ No newline at end of file diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index 3a410ed..ff58ab0 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -26,11 +26,15 @@ def start(self): # Register the file descriptor for read event self.loop.add_reader(self._master, self.read_handler) - def stop(self): + def stop_loop(self): logging.info('Stopping UART event loop') + self._send_queue.put_nowait(None) + + def remove(self): # Unregister the fd self.loop.remove_reader(self._master) os.remove(self.symlink) + logging.info(f'UART reader and symlink removed') def read_handler(self): @@ -48,5 +52,7 @@ def queue_write(self, value: bytes): async def write_loop(self): while True: data = await self._send_queue.get() + if data == None: + break # Let future end on shutdown logging.debug(f'Write: {data}') os.write(self._master, data) From 415b5939932fafec8ead8cba8c62db1862b636c8 Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 15:59:01 +0200 Subject: [PATCH 14/35] Replace acronyms in log messages --- ble_serial/ble_interface.py | 4 ++-- ble_serial/serial/linux_pty.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py index de5526b..e8fba34 100644 --- a/ble_serial/ble_interface.py +++ b/ble_serial/ble_interface.py @@ -58,7 +58,7 @@ async def send_loop(self): await self.dev.write_gatt_char(self.write_char, data) def stop_loop(self): - logging.info('Stopping BLE event loop') + logging.info('Stopping Bluetooth event loop') self._send_queue.put_nowait(None) async def disconnect(self): @@ -66,7 +66,7 @@ async def disconnect(self): if hasattr(self, 'read_char'): await self.dev.stop_notify(self.read_char) await self.dev.disconnect() - logging.info('BT disconnected') + logging.info('Bluetooth disconnected') def queue_send(self, data: bytes): self._send_queue.put_nowait(data) diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index ff58ab0..27af69e 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -22,19 +22,18 @@ def set_receiver(self, callback): def start(self): assert self._cb, 'Receiver must be set before start!' - logging.info('Starting UART event loop') # Register the file descriptor for read event self.loop.add_reader(self._master, self.read_handler) def stop_loop(self): - logging.info('Stopping UART event loop') + logging.info('Stopping serial event loop') self._send_queue.put_nowait(None) def remove(self): # Unregister the fd self.loop.remove_reader(self._master) os.remove(self.symlink) - logging.info(f'UART reader and symlink removed') + logging.info(f'Serial reader and symlink removed') def read_handler(self): From 4d19b46e9e82df779c622d0e1f650a96a773c0d7 Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 16:18:33 +0200 Subject: [PATCH 15/35] Fix KeyboardInterrupt stacktrace --- ble_serial/__main__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index c7ceee4..43c5083 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -36,7 +36,11 @@ def __init__(self): ) logging.getLogger('bleak').level = logging.INFO - asyncio.run(self.run()) + try: + asyncio.run(self.run()) + # KeyboardInterrupt causes bluetooth to disconnect, but still a exception would be printed here + except KeyboardInterrupt as e: + logging.debug('Exit due to KeyboardInterrupt') async def run(self): args = self.args @@ -61,8 +65,9 @@ async def run(self): except BleakError as e: logging.warning(f'Bluetooth connection failed: {e}') - except KeyboardInterrupt: - logging.info('Keyboard interrupt received') + ### KeyboardInterrupts are now received on asyncio.run() + # except KeyboardInterrupt: + # logging.info('Keyboard interrupt received') except Exception as e: logging.error(f'Unexpected Error: {e}') finally: From a4b49682799778768bc729b069c6d1e69ffa880a Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 16:20:37 +0200 Subject: [PATCH 16/35] Change py requires to 3.7 because of asyncio --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c7e831..335a7a9 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - python_requires='>=3.6', + python_requires='>=3.7', install_requires=[ 'bleak >= 0.11.0' ], From eee8cc55f2903476b1f7b61212eb81cf3a1cb682 Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 16:56:12 +0200 Subject: [PATCH 17/35] Add MTU and fix other ble params --- ble_serial/__main__.py | 6 ++++-- ble_serial/ble_interface.py | 2 +- ble_serial/serial/linux_pty.py | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 43c5083..82cc5bb 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -15,8 +15,10 @@ def __init__(self): help='BLE device address to connect (hex format, can be seperated by colons)') parser.add_argument('-t', '--address-type', dest='addr_type', required=False, choices=['public', 'random'], default='public', help='BLE address type, either public or random') - parser.add_argument('-i', '--interface', dest='adapter', required=False, default='0', + parser.add_argument('-i', '--interface', dest='adapter', required=False, default='hci0', help='BLE host adapter number to use') + parser.add_argument('-m', '--mtu', dest='mtu', required=False, default=20, type=int, + help='Max. bluetooth packet data size in bytes used for sending') parser.add_argument('-w', '--write-uuid', dest='write_uuid', required=False, help='The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out') parser.add_argument('-l', '--log', dest='filename', required=False, @@ -47,7 +49,7 @@ async def run(self): loop = asyncio.get_event_loop() loop.set_exception_handler(self.excp_handler) try: - self.uart = UART(args.port, loop) + self.uart = UART(args.port, loop, args.mtu) self.bt = BLE_interface() if args.filename: self.log = FS_log(args.filename, args.binlog) diff --git a/ble_serial/ble_interface.py b/ble_serial/ble_interface.py index e8fba34..7762447 100644 --- a/ble_serial/ble_interface.py +++ b/ble_serial/ble_interface.py @@ -7,7 +7,7 @@ class BLE_interface(): async def start(self, addr_str, addr_type, adapter, write_uuid, read_uuid,): - self.dev = BleakClient(addr_str) # addr_type only on Windows?, adapter in kwargs + self.dev = BleakClient(addr_str, adapter=adapter, address_type=addr_type) # address_type used only in Windows .NET currently self._send_queue = asyncio.Queue() self.dev.set_disconnected_callback(self.handle_disconnect) logging.info(f'Trying to connect with {addr_str}') diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index 27af69e..e4c7293 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -2,8 +2,9 @@ import os, pty, tty, termios class UART(): - def __init__(self, symlink: str, ev_loop: asyncio.AbstractEventLoop): + def __init__(self, symlink: str, ev_loop: asyncio.AbstractEventLoop, mtu: int): self.loop = ev_loop + self.mtu = mtu self._send_queue = asyncio.Queue() master, slave = pty.openpty() @@ -41,7 +42,7 @@ def read_handler(self): self._cb(data) def read_sync(self): - value = os.read(self._master, 20) + value = os.read(self._master, self.mtu) logging.debug(f'Read: {value}') return value From 4eabb0fd101c26265857db61e66e974e889a8edc Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 7 Apr 2021 02:05:03 +0200 Subject: [PATCH 18/35] Create abstract serial interface --- ble_serial/serial/interface.py | 27 +++++++++++++++++++++++++++ ble_serial/serial/linux_pty.py | 1 + 2 files changed, 28 insertions(+) create mode 100644 ble_serial/serial/interface.py diff --git a/ble_serial/serial/interface.py b/ble_serial/serial/interface.py new file mode 100644 index 0000000..21f7a1d --- /dev/null +++ b/ble_serial/serial/interface.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod + +class ISerial(ABC): + + @abstractmethod + def start(self): + pass + + @abstractmethod + def set_receiver(self, callback): + pass + + @abstractmethod + def queue_write(self, value: bytes): + pass + + @abstractmethod + async def write_loop(self): + pass + + @abstractmethod + def stop_loop(self): + pass + + @abstractmethod + def remove(self): + pass \ No newline at end of file diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index e4c7293..7bfbf0a 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -1,3 +1,4 @@ +from ble_serial.serial.interface import ISerial import asyncio, logging import os, pty, tty, termios From 25bf586755d73bc19009f721a47d2cbd89e25c31 Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 7 Apr 2021 02:12:18 +0200 Subject: [PATCH 19/35] Rename uart loop method --- ble_serial/__main__.py | 2 +- ble_serial/serial/interface.py | 2 +- ble_serial/serial/linux_pty.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 82cc5bb..bb899c7 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -62,7 +62,7 @@ async def run(self): self.uart.start() await self.bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) logging.info('Running main loop!') - self.main_loop = asyncio.gather(self.bt.send_loop(), self.uart.write_loop()) + self.main_loop = asyncio.gather(self.bt.send_loop(), self.uart.run_loop()) await self.main_loop except BleakError as e: diff --git a/ble_serial/serial/interface.py b/ble_serial/serial/interface.py index 21f7a1d..6d14ec5 100644 --- a/ble_serial/serial/interface.py +++ b/ble_serial/serial/interface.py @@ -15,7 +15,7 @@ def queue_write(self, value: bytes): pass @abstractmethod - async def write_loop(self): + async def run_loop(self): pass @abstractmethod diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/serial/linux_pty.py index 7bfbf0a..4d1af6b 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/serial/linux_pty.py @@ -2,7 +2,7 @@ import asyncio, logging import os, pty, tty, termios -class UART(): +class UART(ISerial): def __init__(self, symlink: str, ev_loop: asyncio.AbstractEventLoop, mtu: int): self.loop = ev_loop self.mtu = mtu @@ -50,7 +50,7 @@ def read_sync(self): def queue_write(self, value: bytes): self._send_queue.put_nowait(value) - async def write_loop(self): + async def run_loop(self): while True: data = await self._send_queue.get() if data == None: From 31cf61df0d0305796e83be90f03e92a8b6482bd0 Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 7 Apr 2021 02:27:48 +0200 Subject: [PATCH 20/35] Add dummy serial module --- ble_serial/__main__.py | 3 ++- ble_serial/serial/print_dummy.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 ble_serial/serial/print_dummy.py diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index bb899c7..358a313 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,5 +1,6 @@ import logging, sys, argparse, time, asyncio -from ble_serial.serial.linux_pty import UART +# from ble_serial.serial.linux_pty import UART +from ble_serial.serial.print_dummy import Dummy as UART from ble_serial.ble_interface import BLE_interface from ble_serial.fs_log import FS_log, Direction from bleak.exc import BleakError diff --git a/ble_serial/serial/print_dummy.py b/ble_serial/serial/print_dummy.py new file mode 100644 index 0000000..c1eff21 --- /dev/null +++ b/ble_serial/serial/print_dummy.py @@ -0,0 +1,31 @@ +from ble_serial.serial.interface import ISerial +import asyncio + +TEST_DATA = b'Test UART out\n' + +class Dummy(ISerial): + def __init__(self, symlink: str, ev_loop: asyncio.AbstractEventLoop, mtu: int): + print(f'{symlink=} {mtu=}') + pass + + def start(self): + pass + + def set_receiver(self, callback): + self._cb = callback + pass + + def queue_write(self, value: bytes): + print(f'Dummy write queue: {value}') + + async def run_loop(self): + while True: + await asyncio.sleep(5) + self._cb(TEST_DATA) + print(f'Dummy read callback: {TEST_DATA}') + + def stop_loop(self): + pass + + def remove(self): + pass \ No newline at end of file From 4a86bb4a0db289be69c415981f469fe3016c7420 Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 7 Apr 2021 14:57:06 +0200 Subject: [PATCH 21/35] Add priv com0com setup script --- ble_serial/serial/windows_priv_setupc.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 ble_serial/serial/windows_priv_setupc.py diff --git a/ble_serial/serial/windows_priv_setupc.py b/ble_serial/serial/windows_priv_setupc.py new file mode 100644 index 0000000..cf199ba --- /dev/null +++ b/ble_serial/serial/windows_priv_setupc.py @@ -0,0 +1,49 @@ +import os +import subprocess, re + +# For compatibility: +# https://support.microsoft.com/en-us/topic/howto-specify-serial-ports-larger-than-com9-db9078a5-b7b6-bf00-240f-f749ebfd913e +PORT_USER = 'COM56' +PORT_INTERNAL = 'BLE3' + +SETUP_PATH = 'C:/Program Files (x86)/com0com/' +BIN = 'setupc.exe' + +def cd(path: str): + os.chdir(path) + +def check_list(port: str): + print(f'\n> Checking port list for {port}') + p = subprocess.run([BIN, 'list'], check=True, capture_output=True) + stdout = p.stdout.decode() + print(stdout) + return re.compile(rf'\w* PortName={port}').search(stdout) + +def install(): + print('> Trying to create port pair') + p = subprocess.run([BIN, 'install', 'PortName='+PORT_USER, 'PortName='+PORT_INTERNAL], + check=True, capture_output=True) + stdout = p.stdout.decode() + print(stdout) + +if __name__ == "__main__": + try: + # com0com needs to read inf from install path + cd(SETUP_PATH) + done = check_list(PORT_INTERNAL) + if done: + print(f'Found: {done.group(0)}') + print('Setup already done!') + else: + print(f'{PORT_INTERNAL} port does not exist') + + if check_list(PORT_USER): + raise Exception(f'Error: user port {PORT_USER} already in use') + + install() + print('Setup done!') + except Exception as e: + print(e) + finally: + # Keep CMD open to show results + input('\nHit any key to close') From ea14fd598b1a640dbe69fb9963153bdc2a7e3ae2 Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 7 Apr 2021 15:03:45 +0200 Subject: [PATCH 22/35] Add windows serial class with UAC based com0com setup --- ble_serial/serial/windows_com0com.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ble_serial/serial/windows_com0com.py diff --git a/ble_serial/serial/windows_com0com.py b/ble_serial/serial/windows_com0com.py new file mode 100644 index 0000000..a8d12da --- /dev/null +++ b/ble_serial/serial/windows_com0com.py @@ -0,0 +1,36 @@ +from ble_serial.serial.interface import ISerial +from ble_serial.serial.windows_priv_setupc import PORT_INTERNAL +import serial + +class COM(ISerial): + def __init__(self): + self.serial = serial.Serial("\\\\.\\BLE") + + def start(self): + pass + + def set_receiver(self, callback): + pass + + def queue_write(self, value: bytes): + pass + + async def run_loop(self): + pass + + def stop_loop(self): + pass + + def remove(self): + pass + +def run_setup(): + import ctypes, sys, os + + script_path = os.path.dirname(__file__) + res = ctypes.windll.shell32.ShellExecuteW(None, "runas", + sys.executable, f'{script_path}\\windows_priv_setupc.py', None, 1) + print('OK' if res == 42 else 'Error: higher privileges required') + +if __name__ == "__main__": + run_setup() \ No newline at end of file From 568a0983730aebbb39cdcac31af5839b2493c5d8 Mon Sep 17 00:00:00 2001 From: Jake Date: Thu, 8 Apr 2021 00:56:04 +0200 Subject: [PATCH 23/35] Add setup com0com command --- ble_serial/serial/windows_com0com.py | 10 ---------- ble_serial/setup_com0com/__init__.py | 18 ++++++++++++++++++ ble_serial/setup_com0com/__main__.py | 4 ++++ .../windows_priv_setupc.py | 15 ++++++++------- setup.py | 12 +++++++++--- 5 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 ble_serial/setup_com0com/__init__.py create mode 100644 ble_serial/setup_com0com/__main__.py rename ble_serial/{serial => setup_com0com}/windows_priv_setupc.py (85%) diff --git a/ble_serial/serial/windows_com0com.py b/ble_serial/serial/windows_com0com.py index a8d12da..31ea19d 100644 --- a/ble_serial/serial/windows_com0com.py +++ b/ble_serial/serial/windows_com0com.py @@ -24,13 +24,3 @@ def stop_loop(self): def remove(self): pass -def run_setup(): - import ctypes, sys, os - - script_path = os.path.dirname(__file__) - res = ctypes.windll.shell32.ShellExecuteW(None, "runas", - sys.executable, f'{script_path}\\windows_priv_setupc.py', None, 1) - print('OK' if res == 42 else 'Error: higher privileges required') - -if __name__ == "__main__": - run_setup() \ No newline at end of file diff --git a/ble_serial/setup_com0com/__init__.py b/ble_serial/setup_com0com/__init__.py new file mode 100644 index 0000000..b7dfeb1 --- /dev/null +++ b/ble_serial/setup_com0com/__init__.py @@ -0,0 +1,18 @@ +import argparse + +def run_setup(path: str): + import ctypes, sys, os + + script_path = os.path.dirname(__file__) + res = ctypes.windll.shell32.ShellExecuteW(None, "runas", + sys.executable, f'{script_path}\\windows_priv_setupc.py "{path}"', None, 1) + print('OK' if res == 42 else 'Error: higher privileges required') + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description='Setup required COM port pair') + parser.add_argument('--install-path', default='C:/Program Files (x86)/com0com/', + help='Installation directory of the null modem emulator') + args = parser.parse_args() + + run_setup(args.install_path) diff --git a/ble_serial/setup_com0com/__main__.py b/ble_serial/setup_com0com/__main__.py new file mode 100644 index 0000000..6590909 --- /dev/null +++ b/ble_serial/setup_com0com/__main__.py @@ -0,0 +1,4 @@ +from ble_serial.setup_com0com import main + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ble_serial/serial/windows_priv_setupc.py b/ble_serial/setup_com0com/windows_priv_setupc.py similarity index 85% rename from ble_serial/serial/windows_priv_setupc.py rename to ble_serial/setup_com0com/windows_priv_setupc.py index cf199ba..63c4ed8 100644 --- a/ble_serial/serial/windows_priv_setupc.py +++ b/ble_serial/setup_com0com/windows_priv_setupc.py @@ -1,15 +1,17 @@ -import os +import os, sys import subprocess, re # For compatibility: # https://support.microsoft.com/en-us/topic/howto-specify-serial-ports-larger-than-com9-db9078a5-b7b6-bf00-240f-f749ebfd913e -PORT_USER = 'COM56' -PORT_INTERNAL = 'BLE3' +PORT_USER = 'COM9' +PORT_INTERNAL = 'BLE' -SETUP_PATH = 'C:/Program Files (x86)/com0com/' BIN = 'setupc.exe' -def cd(path: str): +def cd_to_install(): + # com0com needs to read inf from install path + path = sys.argv[1] + print(f'Changing into {path}') os.chdir(path) def check_list(port: str): @@ -28,8 +30,7 @@ def install(): if __name__ == "__main__": try: - # com0com needs to read inf from install path - cd(SETUP_PATH) + cd_to_install() done = check_list(PORT_INTERNAL) if done: print(f'Found: {done.group(0)}') diff --git a/setup.py b/setup.py index 335a7a9..0145ebb 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -import setuptools +import setuptools, platform with open("README.md", "r") as fh: long_description = fh.read() @@ -12,7 +12,11 @@ long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/Jakeler/ble-serial", - packages=["ble_serial"], + packages=[ + "ble_serial", + "ble_serial.serial", + "ble_serial.setup_com0com" + ], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", @@ -25,7 +29,9 @@ entry_points={ 'console_scripts': [ 'ble-scan=ble_serial.scan:main', - 'ble-serial=ble_serial.__main__:main', + 'ble-serial=ble_serial.__main__:Main', ] + + ['ble-setup=ble_serial.setup_com0com:main'] + if platform.system() == "Windows" else [] }, ) From 4f0f577d145e40f7a28f341f89cc2edfb2efc356 Mon Sep 17 00:00:00 2001 From: Jake Date: Thu, 8 Apr 2021 00:57:23 +0200 Subject: [PATCH 24/35] Add pyserial dependency for windows --- requirements.txt | 4 +++- setup.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 485bc30..1925536 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -bleak >= 0.11.0 \ No newline at end of file +bleak >= 0.11.0 +pyserial >= 3.4.0 ;platform_system == "Windows" +aioserial >= 1.3.0 ;platform_system == "Windows" \ No newline at end of file diff --git a/setup.py b/setup.py index 0145ebb..8376df2 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,9 @@ ], python_requires='>=3.7', install_requires=[ - 'bleak >= 0.11.0' + 'bleak >= 0.11.0', + 'pyserial >= 3.4.0 ;platform_system == "Windows"', + 'aioserial >= 1.3.0 ;platform_system == "Windows"', ], entry_points={ 'console_scripts': [ From 3f33b7f8d6884e0c271a848ef0ebedd7c18c86a4 Mon Sep 17 00:00:00 2001 From: Jake Date: Thu, 8 Apr 2021 01:09:32 +0200 Subject: [PATCH 25/35] Rename serial submodule to ports --- ble_serial/__main__.py | 4 ++-- ble_serial/{serial => ports}/interface.py | 0 ble_serial/{serial => ports}/linux_pty.py | 2 +- ble_serial/{serial => ports}/print_dummy.py | 2 +- ble_serial/{serial => ports}/windows_com0com.py | 4 ++-- setup.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename ble_serial/{serial => ports}/interface.py (100%) rename ble_serial/{serial => ports}/linux_pty.py (97%) rename ble_serial/{serial => ports}/print_dummy.py (93%) rename ble_serial/{serial => ports}/windows_com0com.py (76%) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 358a313..39b1cad 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,6 +1,6 @@ import logging, sys, argparse, time, asyncio -# from ble_serial.serial.linux_pty import UART -from ble_serial.serial.print_dummy import Dummy as UART +# from ble_serial.ports.linux_pty import UART +from ble_serial.ports.print_dummy import Dummy as UART from ble_serial.ble_interface import BLE_interface from ble_serial.fs_log import FS_log, Direction from bleak.exc import BleakError diff --git a/ble_serial/serial/interface.py b/ble_serial/ports/interface.py similarity index 100% rename from ble_serial/serial/interface.py rename to ble_serial/ports/interface.py diff --git a/ble_serial/serial/linux_pty.py b/ble_serial/ports/linux_pty.py similarity index 97% rename from ble_serial/serial/linux_pty.py rename to ble_serial/ports/linux_pty.py index 4d1af6b..3022ba4 100644 --- a/ble_serial/serial/linux_pty.py +++ b/ble_serial/ports/linux_pty.py @@ -1,4 +1,4 @@ -from ble_serial.serial.interface import ISerial +from ble_serial.ports.interface import ISerial import asyncio, logging import os, pty, tty, termios diff --git a/ble_serial/serial/print_dummy.py b/ble_serial/ports/print_dummy.py similarity index 93% rename from ble_serial/serial/print_dummy.py rename to ble_serial/ports/print_dummy.py index c1eff21..5b0ac8f 100644 --- a/ble_serial/serial/print_dummy.py +++ b/ble_serial/ports/print_dummy.py @@ -1,4 +1,4 @@ -from ble_serial.serial.interface import ISerial +from ble_serial.ports.interface import ISerial import asyncio TEST_DATA = b'Test UART out\n' diff --git a/ble_serial/serial/windows_com0com.py b/ble_serial/ports/windows_com0com.py similarity index 76% rename from ble_serial/serial/windows_com0com.py rename to ble_serial/ports/windows_com0com.py index 31ea19d..9a50827 100644 --- a/ble_serial/serial/windows_com0com.py +++ b/ble_serial/ports/windows_com0com.py @@ -1,5 +1,5 @@ -from ble_serial.serial.interface import ISerial -from ble_serial.serial.windows_priv_setupc import PORT_INTERNAL +from ble_serial.ports.interface import ISerial +from ble_serial.ports.windows_priv_setupc import PORT_INTERNAL import serial class COM(ISerial): diff --git a/setup.py b/setup.py index 8376df2..345216d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ url="https://github.com/Jakeler/ble-serial", packages=[ "ble_serial", - "ble_serial.serial", + "ble_serial.ports", "ble_serial.setup_com0com" ], classifiers=[ From 7c1952a59f83712e6ab0c362be0f59326bd35d8c Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 12 Apr 2021 23:39:09 +0200 Subject: [PATCH 26/35] Use correct windows port --- ble_serial/ports/windows_com0com.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ble_serial/ports/windows_com0com.py b/ble_serial/ports/windows_com0com.py index 9a50827..3cd5253 100644 --- a/ble_serial/ports/windows_com0com.py +++ b/ble_serial/ports/windows_com0com.py @@ -1,10 +1,10 @@ from ble_serial.ports.interface import ISerial -from ble_serial.ports.windows_priv_setupc import PORT_INTERNAL -import serial +from ble_serial.setup_com0com.windows_priv_setupc import PORT_INTERNAL +import serial # pyserial class COM(ISerial): def __init__(self): - self.serial = serial.Serial("\\\\.\\BLE") + self.serial = serial.Serial(f"\\\\.\\{PORT_INTERNAL}") def start(self): pass From 76a4c41578e77c56dd56b54e9c84290035988a9c Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 12 Apr 2021 23:39:09 +0200 Subject: [PATCH 27/35] Fix com0com setup regex matched to much --- ble_serial/setup_com0com/windows_priv_setupc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ble_serial/setup_com0com/windows_priv_setupc.py b/ble_serial/setup_com0com/windows_priv_setupc.py index 63c4ed8..82f16ef 100644 --- a/ble_serial/setup_com0com/windows_priv_setupc.py +++ b/ble_serial/setup_com0com/windows_priv_setupc.py @@ -19,7 +19,7 @@ def check_list(port: str): p = subprocess.run([BIN, 'list'], check=True, capture_output=True) stdout = p.stdout.decode() print(stdout) - return re.compile(rf'\w* PortName={port}').search(stdout) + return re.compile(rf'\w* PortName={port}$', flags=re.MULTILINE).search(stdout) def install(): print('> Trying to create port pair') From 362d3b626dd1c461cdd81650b3c3fecec2010433 Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 13 Apr 2021 02:27:46 +0200 Subject: [PATCH 28/35] Implement windows pyserial Threadpool based with asyncio --- ble_serial/__main__.py | 2 +- ble_serial/ports/windows_com0com.py | 50 +++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 39b1cad..e06f89a 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,6 +1,6 @@ import logging, sys, argparse, time, asyncio # from ble_serial.ports.linux_pty import UART -from ble_serial.ports.print_dummy import Dummy as UART +from ble_serial.ports.windows_com0com import COM as UART from ble_serial.ble_interface import BLE_interface from ble_serial.fs_log import FS_log, Direction from bleak.exc import BleakError diff --git a/ble_serial/ports/windows_com0com.py b/ble_serial/ports/windows_com0com.py index 3cd5253..7fd4c6d 100644 --- a/ble_serial/ports/windows_com0com.py +++ b/ble_serial/ports/windows_com0com.py @@ -1,26 +1,58 @@ from ble_serial.ports.interface import ISerial from ble_serial.setup_com0com.windows_priv_setupc import PORT_INTERNAL -import serial # pyserial +import asyncio, logging +from serial import Serial # pyserial +from queue import Queue, Empty +from concurrent.futures import ThreadPoolExecutor class COM(ISerial): - def __init__(self): - self.serial = serial.Serial(f"\\\\.\\{PORT_INTERNAL}") + def __init__(self, _, ev_loop: asyncio.AbstractEventLoop, mtu: int): + self.alive = True # to stop executor threads + self.loop = ev_loop + self.mtu = mtu + self.tx_queue = Queue() def start(self): - pass + self.serial = Serial(f"\\\\.\\{PORT_INTERNAL}") def set_receiver(self, callback): - pass + self._cb = callback def queue_write(self, value: bytes): - pass + self.tx_queue.put(value) async def run_loop(self): - pass + pool = ThreadPoolExecutor(max_workers=2) + rx = self.loop.run_in_executor(pool, self._run_rx) + tx = self.loop.run_in_executor(pool, self._run_tx) + return asyncio.gather(rx, tx) + + def _run_tx(self): + while self.alive and self.serial.is_open: + try: + data = self.tx_queue.get(block=True, timeout=3) + logging.debug(f'Write: {data}') + self.serial.write(data) + except Empty as e: + logging.debug('TX queue timeout: was empty') + + logging.debug(f'TX loop ended {self.alive=} {self.serial.is_open=}') + + def _run_rx(self): + # based on ReaderThread(threading.Thread) from: + # https://github.com/pyserial/pyserial/blob/master/serial/threaded/__init__.py + while self.alive and self.serial.is_open: + n = min(self.mtu, self.serial.in_waiting) or 1 # request at least 1 to block + data = self.serial.read(n) + logging.debug(f'Read: {data}') + self.loop.call_soon_threadsafe(self._cb, data) # needed as asyncio.Queue is not thread safe + + logging.debug(f'RX loop ended {self.alive=} {self.serial.is_open=}') def stop_loop(self): - pass + self.alive = False + logging.info('Stopping RX+TX loop') def remove(self): - pass + self.serial.close() From d173f80968241c15ead4f97897a6e8759f9ac7f3 Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 13 Apr 2021 02:46:52 +0200 Subject: [PATCH 29/35] More efficient pyserial multi byte read --- ble_serial/ports/windows_com0com.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ble_serial/ports/windows_com0com.py b/ble_serial/ports/windows_com0com.py index 7fd4c6d..955ec7f 100644 --- a/ble_serial/ports/windows_com0com.py +++ b/ble_serial/ports/windows_com0com.py @@ -42,8 +42,9 @@ def _run_rx(self): # based on ReaderThread(threading.Thread) from: # https://github.com/pyserial/pyserial/blob/master/serial/threaded/__init__.py while self.alive and self.serial.is_open: - n = min(self.mtu, self.serial.in_waiting) or 1 # request at least 1 to block - data = self.serial.read(n) + data = self.serial.read(1) # request 1 to block + n = min(self.mtu - 1, self.serial.in_waiting) # read the remaning, can be 0 + data += self.serial.read(n) logging.debug(f'Read: {data}') self.loop.call_soon_threadsafe(self._cb, data) # needed as asyncio.Queue is not thread safe From d7fa3a761acac68d9676a0bab43d324cd8e3991d Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 13 Apr 2021 03:36:45 +0200 Subject: [PATCH 30/35] Create another main --- ble_serial/__init__.py | 1 + ble_serial/__main__.py | 93 +--------------------------------------- ble_serial/main.py | 97 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 8 ++-- 4 files changed, 103 insertions(+), 96 deletions(-) create mode 100644 ble_serial/main.py diff --git a/ble_serial/__init__.py b/ble_serial/__init__.py index e69de29..ad8fd2f 100644 --- a/ble_serial/__init__.py +++ b/ble_serial/__init__.py @@ -0,0 +1 @@ +from ble_serial.main import Main diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index e06f89a..511afc0 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,95 +1,4 @@ -import logging, sys, argparse, time, asyncio -# from ble_serial.ports.linux_pty import UART -from ble_serial.ports.windows_com0com import COM as UART -from ble_serial.ble_interface import BLE_interface -from ble_serial.fs_log import FS_log, Direction -from bleak.exc import BleakError - -class Main(): - def __init__(self): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Create virtual serial ports from BLE devices.') - - parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', - help='Increase verbosity to log all data going through') - parser.add_argument('-d', '--dev', dest='device', required=True, - help='BLE device address to connect (hex format, can be seperated by colons)') - parser.add_argument('-t', '--address-type', dest='addr_type', required=False, choices=['public', 'random'], default='public', - help='BLE address type, either public or random') - parser.add_argument('-i', '--interface', dest='adapter', required=False, default='hci0', - help='BLE host adapter number to use') - parser.add_argument('-m', '--mtu', dest='mtu', required=False, default=20, type=int, - help='Max. bluetooth packet data size in bytes used for sending') - parser.add_argument('-w', '--write-uuid', dest='write_uuid', required=False, - help='The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out') - parser.add_argument('-l', '--log', dest='filename', required=False, - help='Enable optional logging of all bluetooth traffic to file') - parser.add_argument('-b', '--binary', dest='binlog', required=False, action='store_true', - help='Log data as raw binary, disable transformation to hex. Works only in combination with -l') - parser.add_argument('-p', '--port', dest='port', required=False, default='/tmp/ttyBLE', - help='Symlink to virtual serial port') - parser.add_argument('-r', '--read-uuid', dest='read_uuid', required=False, - help='The GATT characteristic to subscribe to notifications to read the serial data') - self.args = parser.parse_args() - - logging.basicConfig( - format='%(asctime)s.%(msecs)03d | %(levelname)s | %(filename)s: %(message)s', - datefmt='%H:%M:%S', - level=logging.DEBUG if self.args.verbose else logging.INFO - ) - logging.getLogger('bleak').level = logging.INFO - - try: - asyncio.run(self.run()) - # KeyboardInterrupt causes bluetooth to disconnect, but still a exception would be printed here - except KeyboardInterrupt as e: - logging.debug('Exit due to KeyboardInterrupt') - - async def run(self): - args = self.args - loop = asyncio.get_event_loop() - loop.set_exception_handler(self.excp_handler) - try: - self.uart = UART(args.port, loop, args.mtu) - self.bt = BLE_interface() - if args.filename: - self.log = FS_log(args.filename, args.binlog) - self.bt.set_receiver(self.log.middleware(Direction.BLE_IN, self.uart.queue_write)) - self.uart.set_receiver(self.log.middleware(Direction.BLE_OUT, self.bt.queue_send)) - else: - self.bt.set_receiver(self.uart.queue_write) - self.uart.set_receiver(self.bt.queue_send) - - self.uart.start() - await self.bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) - logging.info('Running main loop!') - self.main_loop = asyncio.gather(self.bt.send_loop(), self.uart.run_loop()) - await self.main_loop - - except BleakError as e: - logging.warning(f'Bluetooth connection failed: {e}') - ### KeyboardInterrupts are now received on asyncio.run() - # except KeyboardInterrupt: - # logging.info('Keyboard interrupt received') - except Exception as e: - logging.error(f'Unexpected Error: {e}') - finally: - logging.warning('Shutdown initiated') - if hasattr(self, 'uart'): - self.uart.remove() - if hasattr(self, 'bt'): - await self.bt.disconnect() - if hasattr(self, 'log'): - self.log.finish() - logging.info('Shutdown complete.') - - - def excp_handler(self, loop: asyncio.AbstractEventLoop, context): - # Handles exception from other tasks (inside bleak disconnect, etc) - # loop.default_exception_handler(context) - logging.debug(f'Asyncio execption handler called {context["exception"]}') - self.uart.stop_loop() - self.bt.stop_loop() +from ble_serial import Main if __name__ == '__main__': Main() \ No newline at end of file diff --git a/ble_serial/main.py b/ble_serial/main.py new file mode 100644 index 0000000..0edca91 --- /dev/null +++ b/ble_serial/main.py @@ -0,0 +1,97 @@ +import logging, sys, argparse, time, asyncio +from ble_serial.ports.linux_pty import UART +# from ble_serial.ports.windows_com0com import COM as UART +from ble_serial.ble_interface import BLE_interface +from ble_serial.fs_log import FS_log, Direction +from bleak.exc import BleakError + +class Main(): + def __init__(self): + self.parse_args() + self.setup_logger() + + try: + asyncio.run(self.run()) + # KeyboardInterrupt causes bluetooth to disconnect, but still a exception would be printed here + except KeyboardInterrupt as e: + logging.debug('Exit due to KeyboardInterrupt') + + def setup_logger(self): + logging.basicConfig( + format='%(asctime)s.%(msecs)03d | %(levelname)s | %(filename)s: %(message)s', + datefmt='%H:%M:%S', + level=logging.DEBUG if self.args.verbose else logging.INFO + ) + logging.getLogger('bleak').level = logging.INFO + + def parse_args(self): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description='Create virtual serial ports from BLE devices.') + + parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='Increase verbosity to log all data going through') + parser.add_argument('-d', '--dev', dest='device', required=True, + help='BLE device address to connect (hex format, can be seperated by colons)') + parser.add_argument('-t', '--address-type', dest='addr_type', required=False, choices=['public', 'random'], default='public', + help='BLE address type, either public or random') + parser.add_argument('-i', '--interface', dest='adapter', required=False, default='hci0', + help='BLE host adapter number to use') + parser.add_argument('-m', '--mtu', dest='mtu', required=False, default=20, type=int, + help='Max. bluetooth packet data size in bytes used for sending') + parser.add_argument('-w', '--write-uuid', dest='write_uuid', required=False, + help='The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out') + parser.add_argument('-l', '--log', dest='filename', required=False, + help='Enable optional logging of all bluetooth traffic to file') + parser.add_argument('-b', '--binary', dest='binlog', required=False, action='store_true', + help='Log data as raw binary, disable transformation to hex. Works only in combination with -l') + parser.add_argument('-p', '--port', dest='port', required=False, default='/tmp/ttyBLE', + help='Symlink to virtual serial port') + parser.add_argument('-r', '--read-uuid', dest='read_uuid', required=False, + help='The GATT characteristic to subscribe to notifications to read the serial data') + self.args = parser.parse_args() + + async def run(self): + args = self.args + loop = asyncio.get_event_loop() + loop.set_exception_handler(self.excp_handler) + try: + self.uart = UART(args.port, loop, args.mtu) + self.bt = BLE_interface() + if args.filename: + self.log = FS_log(args.filename, args.binlog) + self.bt.set_receiver(self.log.middleware(Direction.BLE_IN, self.uart.queue_write)) + self.uart.set_receiver(self.log.middleware(Direction.BLE_OUT, self.bt.queue_send)) + else: + self.bt.set_receiver(self.uart.queue_write) + self.uart.set_receiver(self.bt.queue_send) + + self.uart.start() + await self.bt.start(args.device, args.addr_type, args.adapter, args.write_uuid, args.read_uuid) + logging.info('Running main loop!') + self.main_loop = asyncio.gather(self.bt.send_loop(), self.uart.run_loop()) + await self.main_loop + + except BleakError as e: + logging.warning(f'Bluetooth connection failed: {e}') + ### KeyboardInterrupts are now received on asyncio.run() + # except KeyboardInterrupt: + # logging.info('Keyboard interrupt received') + except Exception as e: + logging.error(f'Unexpected Error: {e}') + finally: + logging.warning('Shutdown initiated') + if hasattr(self, 'uart'): + self.uart.remove() + if hasattr(self, 'bt'): + await self.bt.disconnect() + if hasattr(self, 'log'): + self.log.finish() + logging.info('Shutdown complete.') + + + def excp_handler(self, loop: asyncio.AbstractEventLoop, context): + # Handles exception from other tasks (inside bleak disconnect, etc) + # loop.default_exception_handler(context) + logging.debug(f'Asyncio execption handler called {context["exception"]}') + self.uart.stop_loop() + self.bt.stop_loop() diff --git a/setup.py b/setup.py index 345216d..8aaaecb 100644 --- a/setup.py +++ b/setup.py @@ -31,9 +31,9 @@ entry_points={ 'console_scripts': [ 'ble-scan=ble_serial.scan:main', - 'ble-serial=ble_serial.__main__:Main', - ] - + ['ble-setup=ble_serial.setup_com0com:main'] - if platform.system() == "Windows" else [] + 'ble-serial=ble_serial:Main', + ] + ( + ['ble-setup=ble_serial.setup_com0com:main'] if platform.system() == "Windows" else [] + ) }, ) From 0c306efe99e3273b20f4d699418f90bc4b38bd2e Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 13 Apr 2021 04:01:54 +0200 Subject: [PATCH 31/35] Platform specific startup --- ble_serial/__init__.py | 9 ++++++++- ble_serial/__main__.py | 4 ++-- ble_serial/main.py | 20 +++++++++++--------- setup.py | 2 +- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/ble_serial/__init__.py b/ble_serial/__init__.py index ad8fd2f..4ceec4b 100644 --- a/ble_serial/__init__.py +++ b/ble_serial/__init__.py @@ -1 +1,8 @@ -from ble_serial.main import Main +import platform + +if platform.system() == 'Linux': + from ble_serial.ports.linux_pty import UART as platform_uart +elif platform.system() == 'Windows': + from ble_serial.ports.windows_com0com import COM as platform_uart +else: + raise Exception('Platform not supported!') \ No newline at end of file diff --git a/ble_serial/__main__.py b/ble_serial/__main__.py index 511afc0..a6fb9f8 100644 --- a/ble_serial/__main__.py +++ b/ble_serial/__main__.py @@ -1,4 +1,4 @@ -from ble_serial import Main +from ble_serial.main import launch if __name__ == '__main__': - Main() \ No newline at end of file + launch() \ No newline at end of file diff --git a/ble_serial/main.py b/ble_serial/main.py index 0edca91..a401a10 100644 --- a/ble_serial/main.py +++ b/ble_serial/main.py @@ -1,17 +1,13 @@ import logging, sys, argparse, time, asyncio -from ble_serial.ports.linux_pty import UART -# from ble_serial.ports.windows_com0com import COM as UART +from bleak.exc import BleakError +from ble_serial import platform_uart as UART from ble_serial.ble_interface import BLE_interface from ble_serial.fs_log import FS_log, Direction -from bleak.exc import BleakError class Main(): - def __init__(self): - self.parse_args() - self.setup_logger() - + def start(self): try: - asyncio.run(self.run()) + asyncio.run(self._run()) # KeyboardInterrupt causes bluetooth to disconnect, but still a exception would be printed here except KeyboardInterrupt as e: logging.debug('Exit due to KeyboardInterrupt') @@ -50,7 +46,7 @@ def parse_args(self): help='The GATT characteristic to subscribe to notifications to read the serial data') self.args = parser.parse_args() - async def run(self): + async def _run(self): args = self.args loop = asyncio.get_event_loop() loop.set_exception_handler(self.excp_handler) @@ -95,3 +91,9 @@ def excp_handler(self, loop: asyncio.AbstractEventLoop, context): logging.debug(f'Asyncio execption handler called {context["exception"]}') self.uart.stop_loop() self.bt.stop_loop() + +def launch(): + m = Main() + m.parse_args() + m.setup_logger() + m.start() \ No newline at end of file diff --git a/setup.py b/setup.py index 8aaaecb..68aa42c 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ entry_points={ 'console_scripts': [ 'ble-scan=ble_serial.scan:main', - 'ble-serial=ble_serial:Main', + 'ble-serial=ble_serial.main:launch', ] + ( ['ble-setup=ble_serial.setup_com0com:main'] if platform.system() == "Windows" else [] ) From 6073b8ea5a43d62616eeeb5a33d1d72b30631d3b Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 14 Apr 2021 16:46:55 +0200 Subject: [PATCH 32/35] Delete old parallelisation experiments --- experiments/async-exp.py | 25 ------------------------- experiments/process-exp.py | 19 ------------------- experiments/thread-exp.py | 19 ------------------- 3 files changed, 63 deletions(-) delete mode 100644 experiments/async-exp.py delete mode 100644 experiments/process-exp.py delete mode 100644 experiments/thread-exp.py diff --git a/experiments/async-exp.py b/experiments/async-exp.py deleted file mode 100644 index 7ff666e..0000000 --- a/experiments/async-exp.py +++ /dev/null @@ -1,25 +0,0 @@ -import asyncio, time - -async def task(name, delay): - for i in range(5): - await asyncio.sleep(delay) - print(name, i) - return f"{name} DONE!" - -async def start(): - r2 = task("One", 1.0) - r1 = task("Two", 0.5) - t2 = asyncio.create_task(r2) - t1 = asyncio.create_task(r1) - # all = asyncio.gather( - # task('Two', 0.5), - # task('One', 1.0) - # ) - for i in range(5): - await asyncio.sleep(2) - print("START", i) - print(await t1, await t2) - # print(await all) - -asyncio.run(start()) - diff --git a/experiments/process-exp.py b/experiments/process-exp.py deleted file mode 100644 index 8cdfd0e..0000000 --- a/experiments/process-exp.py +++ /dev/null @@ -1,19 +0,0 @@ -import time, multiprocessing - -addons = ['Hello', 'world.', 'how', 'are', 'you?'] -position = 0 - -def task(name, delay, mod=False): - global position - - for n in range(5): - for i in range(3_000_000*delay): - pass - if mod: - position = n - print(name, n, addons[position]) - return f"{name} DONE!" - -multiprocessing.Process(target=task, args=("One", 10)).start() -multiprocessing.Process(target=task, args=("Two", 20)).start() -multiprocessing.Process(target=task, args=("Three", 10, True)).start() \ No newline at end of file diff --git a/experiments/thread-exp.py b/experiments/thread-exp.py deleted file mode 100644 index 7f92b17..0000000 --- a/experiments/thread-exp.py +++ /dev/null @@ -1,19 +0,0 @@ -import time, threading - -addons = ['Hello', 'world.', 'how', 'are', 'you?'] -position = 0 - -def task(name, delay, mod=False): - global position - - for n in range(5): - for i in range(3_000_000*delay): - pass - if mod: - position = n - print(name, n, addons[position]) - return f"{name} DONE!" - -threading.Thread(target=task, args=("One", 10)).start() -threading.Thread(target=task, args=("Two", 20)).start() -threading.Thread(target=task, args=("Three", 10, True)).start() \ No newline at end of file From 30ea516a84aa2d8b2a8f2ee8052c44f03c214b1e Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 14 Apr 2021 17:51:21 +0200 Subject: [PATCH 33/35] Update installation + scan readme for bleak --- README.md | 105 +++++++++++++++++++++++++++++------------------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index e5a1d04..1ed892c 100644 --- a/README.md +++ b/README.md @@ -2,55 +2,68 @@ A tool to connect Bluetooth 4.0+ Low Energy to UART modules and normal PCs/laptops/RaspberryPi. It fulfills the same purpose as `rfcomm bind` for the old Bluetooth 2.0, creating a virtual serial port in `/dev/pts/x`, which makes it usable with any terminal or application. -### Installation +## Installation +### Standard (via Python Package Index) The software is written completely in Python and packaged as module, so it can be easily installed with pip: -``` +```console pip install ble-serial -pip install git+https://github.com/edwios/bluepy.git@10f1cee90afb416f139949b86b491e4cfa98c886 ``` -If you are wondering why the second command is required: It depends on the bluepy library, but unfortunately there are [bugs](https://github.com/IanHarvey/bluepy/issues/253) in the original version and there was no development since a year, so it is important to specifically install this fork with a few fixes. Now you should have 2 new scripts: `ble-scan` and the main `ble-serial`. -Note: To be able to do device scans without using `sudo` or root, you must grant the `bluepy-helper` binary additional [capabilities/permissions](https://github.com/IanHarvey/bluepy/issues/313#issuecomment-428324639). Follow the steps outlined below: +On Linux you ready now and can directly jump to the usage section! -Find bluepy-helper (typically ~/.local/lib/python3.6/site-packages/bluepy/bluepy-helper). -Give it permissions so you don't have to run scripts with sudo: -```sh -sudo setcap 'cap_net_raw,cap_net_admin+eip' bluepy-helper` +### From source/local (for developers) +You can clone the repository with: +```console +git clone https://github.com/Jakeler/ble-serial.git ``` -### Finding devices -First make sure the bluetooth adapter is enabled, for example with `bluetoothctl power on`, then the scan function can be used (note: root is required for this step, if you have not added the capabilities above): +Then switch branches, make changes etc... +The package can be started directly with `-m`: +```console +python -m ble_serial ARGUMENTS # Main tool = ble-serial +python -m ble_serial.scan # BLE scan = ble-scan +python -m ble_serial.setup_com0com # Windows only setup = ble-setup ``` -# ble-scan + +Or install it with `pip` from the current directory: +```console +pip install . ``` + +## Usage +### Finding devices +First make sure the bluetooth adapter is enabled, for example with `bluetoothctl power on`, then the scan function can be used: ``` -Discovered device: 20:91:48:4c:4c:54 -> UT61E - JK -... -Found 2 devices! +$ ble-scan -Device 20:91:48:4c:4c:54 (public), RSSI=-58 dB - 01: Flags = 06 - ff: Manufacturer = 484d2091484c4c54 - 16: 16b Service Data = 00b000000000 - 02: Incomplete 16b Services = 0000ffe0-0000-1000-8000-00805f9b34fb - 09: Complete Local Name = UT61E - JK - 0a: Tx Power = 00 +Started BLE scan -Device ... -``` -The output is a list of the recognized nearby devices. After the MAC address it prints out the device name, if it can be resolved. +20:91:48:4C:4C:54 (RSSI=-56): UT61E - JK +... -If there are no devices found it might help to increase the scan time. All discoverable devices must actively send advertisements, to save power the interval of this can be quite long, so then try for example 30 seconds. +Finished BLE scan ``` +The output is a list of the recognized nearby devices. After the MAC address and signal strength it prints out the device name, if it can be resolved. + +If there are no devices found it might help to increase the scan time. All discoverable devices must actively send advertisements, the interval of this can be quite long to save power, so try for example 30 seconds in this case. +```console +$ ble-scan -h +usage: ble-scan [-h] [-t SEC] [-d ADDR] + +Scanner for BLE devices and service/characteristics. + optional arguments: -h, --help show this help message and exit -t SEC, --scan-time SEC Duration of the scan in seconds (default: 5.0) - -d, --deep-scan Try to connect to the devices and read out the service/characteristic UUIDs (default: False) + -d ADDR, --deep-scan ADDR + Try to connect to device and read out service/characteristic UUIDs + (default: None) ``` -On Bluetooth 2.0 there was a "serial port profile", with 4.0 BLE there is unfortunately no standardized mode anymore, every chip manufacturer chooses their own ID to implement the features. + +On Bluetooth 2.0 there was a "serial port profile", with 4.0 - 5.2 (BLE) there is unfortunately no standardized mode anymore, every chip manufacturer chooses their own ID to implement the features. ```py '0000ff02-0000-1000-8000-00805f9b34fb', # LithiumBatteryPCB adapter '0000ffe1-0000-1000-8000-00805f9b34fb', # TI CC245x (HM-10, HM-11) @@ -58,28 +71,24 @@ On Bluetooth 2.0 there was a "serial port profile", with 4.0 BLE there is unfort Some usual IDs are included in ble-serial, these will be tried automatically if nothing is specified. You might skip this part and start directly with the connection. -Otherwise to find the correct ID, use the deep scan option, it will go through the devices and shows all provided interfaces. This scan can take long, especially if there are many devices in the area, so only use it if you want to find the right write characteristic ID. -``` -# ble-scan -d -``` -``` - Device ... - ... - Service: 00001800-0000-1000-8000-00805f9b34fb - Characteristic: 00002a00-0000-1000-8000-00805f9b34fb READ - Characteristic: 00002a01-0000-1000-8000-00805f9b34fb READ - Characteristic: 00002a02-0000-1000-8000-00805f9b34fb READ WRITE - Characteristic: 00002a03-0000-1000-8000-00805f9b34fb READ WRITE - Characteristic: 00002a04-0000-1000-8000-00805f9b34fb READ - Service: 00001801-0000-1000-8000-00805f9b34fb - Characteristic: 00002a05-0000-1000-8000-00805f9b34fb INDICATE - Service: 0000ffe0-0000-1000-8000-00805f9b34fb - Characteristic: 0000ffe1-0000-1000-8000-00805f9b34fb READ WRITE NO RESPONSE NOTIFY -``` -Now in addition to the previous output there are all characteristics listed, grouped into services. The characteristics in the first service starting with `00002a` are not interesting in this case, because they are standard values (for example the device name), if you want to know more look at [this list](https://gist.github.com/sam016/4abe921b5a9ee27f67b3686910293026#file-allgattcharacteristics-java-L57). +Otherwise to find the correct ID, use the deep scan option. It expects a device MAC address, connects to it and reads out all services/characteristic/descriptors: +```console +$ ble-scan -d 20:91:48:4C:4C:54 +Started deep scan of 20:91:48:4C:4C:54 -After the (U)ID the permissions are listed. We are searching for a characteristic that allows writing = sending to the device, the only candidate in here is `0000ffe1-0000-1000-8000-00805f9b34fb` (spoiler: a HM-11 module again). +SERVICE 00001801-0000-1000-8000-00805f9b34fb (Handle: 12): Generic Attribute Profile + CHARACTERISTIC 00002a05-0000-1000-8000-00805f9b34fb (Handle: 13): Service Changed ['indicate'] + DESCRIPTOR 00002902-0000-1000-8000-00805f9b34fb (Handle: 15): Client Characteristic Configuration +SERVICE 0000ffe0-0000-1000-8000-00805f9b34fb (Handle: 16): Vendor specific + CHARACTERISTIC 0000ffe1-0000-1000-8000-00805f9b34fb (Handle: 17): Vendor specific ['read', 'write-without-response', 'notify'] + DESCRIPTOR 00002902-0000-1000-8000-00805f9b34fb (Handle: 19): Client Characteristic Configuration + DESCRIPTOR 00002901-0000-1000-8000-00805f9b34fb (Handle: 20): Characteristic User Description + +Completed deep scan of 20:91:48:4C:4C:54 +``` +Now the interesting parts are the characteristics, grouped into services. The ones belows the first service starting with `00002a` are not interesting in this case, because they are standard values (for example the device name), if you want to know more look at [this list](https://gist.github.com/sam016/4abe921b5a9ee27f67b3686910293026#file-allgattcharacteristics-java-L57). +After the ID, handle and type the permissions are listed in []. We are searching for a characteristic that allows writing = sending to the device, the only candidate in here is `0000ffe1-0000-1000-8000-00805f9b34fb` (spoiler: a HM-11 module again). Same procedure with the read characteristic. ### Connecting a device From f89101dd5799fb892fef96423786ce50821b6699 Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 14 Apr 2021 18:42:49 +0200 Subject: [PATCH 34/35] Finish readme update with windows install section --- README.md | 122 +++++++++++++++++++++++++++++++++++---------- ble_serial/main.py | 2 +- 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 1ed892c..a32ffb7 100644 --- a/README.md +++ b/README.md @@ -13,29 +13,80 @@ Now you should have 2 new scripts: `ble-scan` and the main `ble-serial`. On Linux you ready now and can directly jump to the usage section! -### From source/local (for developers) +### From source (for developers) You can clone the repository with: ```console git clone https://github.com/Jakeler/ble-serial.git ``` Then switch branches, make changes etc... -The package can be started directly with `-m`: +Make sure the dependencies are installed, I recommend to use a virtualenv like this: ```console -python -m ble_serial ARGUMENTS # Main tool = ble-serial -python -m ble_serial.scan # BLE scan = ble-scan -python -m ble_serial.setup_com0com # Windows only setup = ble-setup +$ python -m venv ble-venv +$ source ble-venv/bin/activate +$ pip install -r requirements.txt ``` -Or install it with `pip` from the current directory: +The package can be either started directly with `-m`: ```console -pip install . +$ python -m ble_serial ARGUMENTS # Main tool = ble-serial +$ python -m ble_serial.scan # BLE scan = ble-scan +$ python -m ble_serial.setup_com0com # Windows only setup = ble-setup ``` +Or installed with `pip` from the current directory: +```console +$ pip install . +``` +and started as usual. + +### Additional steps for Windows +Windows does not have a builtin feature to create virtual serial ports (like Linux does), so it is required to install a additional driver. I decided to use the open source `com0com` Null-modem emulator, downloaded from [here](https://sourceforge.net/projects/signed-drivers/files/com0com/v3.0/) as signed version. This is required because unsigned drivers can not be installed anymore. Note that on latest Windows 10 you likely still have to disable secure boot for it to work. + +ble-serial includes the `ble-setup` script to make the `com0com` configuration easier: +``` +ble-setup.exe -h +usage: ble-setup [-h] [--install-path INSTALL_PATH] + +Setup required COM port pair + +optional arguments: + -h, --help show this help message and exit + --install-path INSTALL_PATH + Installation directory of the null modem emulator (default: C:/Program Files (x86)/com0com/) +``` + +It will request administrator privileges (if it does not already have it) and setup the port in another CMD window: +```console +Changing into C:/Program Files (x86)/com0com/ + +> Checking port list for BLE + CNCA0 PortName=- + CNCB0 PortName=- + +BLE port does not exist + +> Checking port list for COM9 + CNCA0 PortName=- + CNCB0 PortName=- + +> Trying to create port pair + CNCA1 PortName=COM9 + CNCB1 PortName=BLE +ComDB: COM9 - logged as "in use" + +Setup done! + +Hit any key to close +``` +As you can see it created the `BLE`<->`COM9` pair. ble-serial will internally connect to `BLE`, users can then send/receive the data on `COM9` + +Otherwise there exist multiple proprietary serial port emulators, these should work too. Just manually create a pair that includes a port named `BLE`. + ## Usage ### Finding devices First make sure the bluetooth adapter is enabled, for example with `bluetoothctl power on`, then the scan function can be used: -``` +```console $ ble-scan Started BLE scan @@ -86,14 +137,22 @@ SERVICE 0000ffe0-0000-1000-8000-00805f9b34fb (Handle: 16): Vendor specific Completed deep scan of 20:91:48:4C:4C:54 ``` -Now the interesting parts are the characteristics, grouped into services. The ones belows the first service starting with `00002a` are not interesting in this case, because they are standard values (for example the device name), if you want to know more look at [this list](https://gist.github.com/sam016/4abe921b5a9ee27f67b3686910293026#file-allgattcharacteristics-java-L57). +Now the interesting parts are the characteristics, grouped into services. The ones belows the first service starting with `00002` are not interesting in this case, because they are standard values (for example the device name), if you want to know more look at [this list](https://gist.github.com/sam016/4abe921b5a9ee27f67b3686910293026#file-allgattcharacteristics-java-L57). -After the ID, handle and type the permissions are listed in []. We are searching for a characteristic that allows writing = sending to the device, the only candidate in here is `0000ffe1-0000-1000-8000-00805f9b34fb` (spoiler: a HM-11 module again). Same procedure with the read characteristic. +After the ID, handle and type the permissions are listed in []. We are searching for a characteristic that allows writing = sending to the device, the only candidate in here is `0000ffe1-0000-1000-8000-00805f9b34fb` (spoiler: a HM-11 module again). +Same procedure with the read characteristic, this modules handle read and write through the same characteristic, but some other chips split it. ### Connecting a device The `ble-serial` tool itself has a few more options: -``` +```console +$ ble_serial -h +usage: __main__.py [-h] [-v] -d DEVICE [-t {public,random}] [-i ADAPTER] [-m MTU] [-w WRITE_UUID] [-l FILENAME] [-b] + [-p PORT] [-r READ_UUID] + +Create virtual serial ports from BLE devices. + +optional arguments: -h, --help show this help message and exit -v, --verbose Increase verbosity to log all data going through (default: False) -d DEVICE, --dev DEVICE @@ -101,26 +160,28 @@ The `ble-serial` tool itself has a few more options: -t {public,random}, --address-type {public,random} BLE address type, either public or random (default: public) -i ADAPTER, --interface ADAPTER - BLE host adapter number to use (default: 0) + BLE host adapter number to use (default: hci0) + -m MTU, --mtu MTU Max. bluetooth packet data size in bytes used for sending (default: 20) -w WRITE_UUID, --write-uuid WRITE_UUID - The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out (default: None) + The GATT characteristic to write the serial data, you might use "ble-scan -d" to find it out + (default: None) -l FILENAME, --log FILENAME Enable optional logging of all bluetooth traffic to file (default: None) - -b, --binary Log data as raw binary, disable transformation to hex. Works only in combination with -l (default: False) + -b, --binary Log data as raw binary, disable transformation to hex. Works only in combination with -l (default: + False) -p PORT, --port PORT Symlink to virtual serial port (default: /tmp/ttyBLE) -r READ_UUID, --read-uuid READ_UUID The GATT characteristic to subscribe to notifications to read the serial data (default: None) ``` Only the device address is always required: -``` +```console $ ble-serial -d 20:91:48:4c:4c:54 -``` -``` -21:02:55.823 | INFO | virtual_serial.py: Slave created on /tmp/ttyBLE -> /dev/pts/8 -21:02:56.410 | INFO | interface.py: Connected device 20:91:48:4c:4c:54 -21:02:56.909 | INFO | interface.py: Receiver set up -21:02:56.909 | INFO | __main__.py: Running main loop! +18:36:09.255 | INFO | linux_pty.py: Slave created on /tmp/ttyBLE -> /dev/pts/7 +18:36:09.255 | INFO | ble_interface.py: Receiver set up +18:36:09.258 | INFO | ble_interface.py: Trying to connect with 20:91:48:4C:4C:54 +18:36:12.291 | INFO | ble_interface.py: Device 20:91:48:4C:4C:54 connected +18:36:12.637 | INFO | main.py: Running main loop! ``` This log shows a successful start, the virtual serial port was opened on `/dev/pts/8`, the number at the end changes, depending on how many pseudo terminals are already open on the system. In addition it creates automatically a symlink to `/tmp/ttyBLE`, so you can easily access it there always on the same file, the default can be changed with the `-p`/`--port` option. @@ -138,18 +199,27 @@ $ ble-serial -d 20:91:48:4c:4c:54 -r 0000ffe1-0000-1000-8000-00805f9b34fb ``` Also there is an option to log all traffic on the link to a text file: -``` +```console $ ble-serial -d 20:91:48:4c:4c:54 -l demo.txt -cat demo.txt -``` -``` +... +$ cat demo.txt 2019-12-09 21:15:53.282805 <- BLE-OUT: 48 65 6c 6c 6f 20 77 6f 72 6c 64 2019-12-09 21:15:53.491681 -> BLE-IN: b0 b0 b0 b0 b0 b0 3b b0 b0 b0 ba b0 0d 8a 2019-12-09 21:15:53.999795 -> BLE-IN: b0 b0 b0 b0 b0 b0 3b b0 b0 b0 ba b0 0d 8a ``` +Per default it is transformed to hex bytes, use `-b`/`--binary` to log raw data. + +You can use `-v` to increase the log verbosity to DEBUG: +``` +18:31:25.136 | DEBUG | ble_interface.py: Received notify from 17: bytearray(b'\xb0\xb0\xb0\xb0\xb0\xb0;\xb0\xb0\xb0\xba\xb0\r\x8a') +18:31:25.136 | DEBUG | linux_pty.py: Write: bytearray(b'\xb0\xb0\xb0\xb0\xb0\xb0;\xb0\xb0\xb0\xba\xb0\r\x8a') + +18:31:25.373 | DEBUG | linux_pty.py: Read: b'hello world' +18:31:25.373 | DEBUG | ble_interface.py: Sending b'hello world' +``` +This will log all traffic going through. Note that everything shows up two times, because it goes through the ble module and then into the serial port and vice versa. As always, i hope it was helpful. If you encounter problems, please use the issue tracker on [GitHub](https://github.com/Jakeler/ble-serial/issues). ### Known limitations -* Higher bitrates: 9600 bit/s is well tested and works fine. 19200 and higher can cause data loss on longer transmissions. * Chromium 73+ based applications, including NW.js/electron desktop apps, for example current Betaflight/INAV Configurator: Connection to the virtual serial port (pty) fails. This is because of explicit whitelisting in chromium. diff --git a/ble_serial/main.py b/ble_serial/main.py index a401a10..9ca3b2b 100644 --- a/ble_serial/main.py +++ b/ble_serial/main.py @@ -35,7 +35,7 @@ def parse_args(self): parser.add_argument('-m', '--mtu', dest='mtu', required=False, default=20, type=int, help='Max. bluetooth packet data size in bytes used for sending') parser.add_argument('-w', '--write-uuid', dest='write_uuid', required=False, - help='The GATT chracteristic to write the serial data, you might use "scan.py -d" to find it out') + help='The GATT characteristic to write the serial data, you might use "ble-scan -d" to find it out') parser.add_argument('-l', '--log', dest='filename', required=False, help='Enable optional logging of all bluetooth traffic to file') parser.add_argument('-b', '--binary', dest='binlog', required=False, action='store_true', From 293b5d3773cde0935f10ca406dcfc1c868a6cf73 Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 14 Apr 2021 18:49:33 +0200 Subject: [PATCH 35/35] Fix console blocks in readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a32ffb7..1a6397c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ It fulfills the same purpose as `rfcomm bind` for the old Bluetooth 2.0, creatin ### Standard (via Python Package Index) The software is written completely in Python and packaged as module, so it can be easily installed with pip: ```console -pip install ble-serial +$ pip install ble-serial ``` Now you should have 2 new scripts: `ble-scan` and the main `ble-serial`. @@ -16,7 +16,7 @@ On Linux you ready now and can directly jump to the usage section! ### From source (for developers) You can clone the repository with: ```console -git clone https://github.com/Jakeler/ble-serial.git +$ git clone https://github.com/Jakeler/ble-serial.git ``` Then switch branches, make changes etc... @@ -44,8 +44,8 @@ and started as usual. Windows does not have a builtin feature to create virtual serial ports (like Linux does), so it is required to install a additional driver. I decided to use the open source `com0com` Null-modem emulator, downloaded from [here](https://sourceforge.net/projects/signed-drivers/files/com0com/v3.0/) as signed version. This is required because unsigned drivers can not be installed anymore. Note that on latest Windows 10 you likely still have to disable secure boot for it to work. ble-serial includes the `ble-setup` script to make the `com0com` configuration easier: -``` -ble-setup.exe -h +```console +> ble-setup.exe -h usage: ble-setup [-h] [--install-path INSTALL_PATH] Setup required COM port pair @@ -57,7 +57,7 @@ optional arguments: ``` It will request administrator privileges (if it does not already have it) and setup the port in another CMD window: -```console +``` Changing into C:/Program Files (x86)/com0com/ > Checking port list for BLE @@ -210,7 +210,7 @@ $ cat demo.txt Per default it is transformed to hex bytes, use `-b`/`--binary` to log raw data. You can use `-v` to increase the log verbosity to DEBUG: -``` +```console 18:31:25.136 | DEBUG | ble_interface.py: Received notify from 17: bytearray(b'\xb0\xb0\xb0\xb0\xb0\xb0;\xb0\xb0\xb0\xba\xb0\r\x8a') 18:31:25.136 | DEBUG | linux_pty.py: Write: bytearray(b'\xb0\xb0\xb0\xb0\xb0\xb0;\xb0\xb0\xb0\xba\xb0\r\x8a')