diff --git a/.github/workflows/package_tests.yml b/.github/workflows/package_tests.yml new file mode 100644 index 000000000..5e503509e --- /dev/null +++ b/.github/workflows/package_tests.yml @@ -0,0 +1,16 @@ +name: Package tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Setup environment + run: source tools/ci.sh && ci_package_tests_setup_micropython + - name: Setup libraries + run: source tools/ci.sh && ci_package_tests_setup_lib + - name: Run tests + run: source tools/ci.sh && ci_package_tests_run diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d590754e9..61a49101e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,15 +102,20 @@ Pages](https://docs.github.com/en/pages): "truthy" value). 5. The settings for GitHub Actions and GitHub Pages features should not need to be changed from the repository defaults, unless you've explicitly disabled - them. + Actions or Pages in your fork. The next time you push commits to a branch in your fork, GitHub Actions will run an additional step in the "Build All Packages" workflow named "Publish Packages -for branch". +for branch". This step runs in *your fork*, but if you open a pull request then +this workflow is not shown in the Pull Request's "Checks". These run in the +upstream repository. Navigate to your fork's Actions tab in order to see +the additional "Publish Packages for branch" step. Anyone can then install these packages as described under [Installing packages -from forks](README.md#installing-packages-from-forks). The exact commands are also -quoted in the GitHub Actions log for the "Publish Packages for branch" step. +from forks](README.md#installing-packages-from-forks). + +The exact command is also quoted in the GitHub Actions log in your fork's +Actions for the "Publish Packages for branch" step of "Build All Packages". #### Opting Back Out diff --git a/micropython/aioespnow/README.md b/micropython/aioespnow/README.md index a68e765af..132bce103 100644 --- a/micropython/aioespnow/README.md +++ b/micropython/aioespnow/README.md @@ -4,7 +4,7 @@ A supplementary module which extends the micropython `espnow` module to provide `asyncio` support. - Asyncio support is available on all ESP32 targets as well as those ESP8266 -boards which include the `uasyncio` module (ie. ESP8266 devices with at least +boards which include the `asyncio` module (ie. ESP8266 devices with at least 2MB flash storage). ## API reference @@ -52,7 +52,7 @@ A small async server example:: ```python import network import aioespnow - import uasyncio as asyncio + import asyncio # A WLAN interface must be active to send()/recv() network.WLAN(network.STA_IF).active(True) diff --git a/micropython/aioespnow/aioespnow.py b/micropython/aioespnow/aioespnow.py index c00c6fb2b..dec925de2 100644 --- a/micropython/aioespnow/aioespnow.py +++ b/micropython/aioespnow/aioespnow.py @@ -1,12 +1,12 @@ # aioespnow module for MicroPython on ESP32 and ESP8266 # MIT license; Copyright (c) 2022 Glenn Moloney @glenn20 -import uasyncio as asyncio +import asyncio import espnow -# Modelled on the uasyncio.Stream class (extmod/stream/stream.py) -# NOTE: Relies on internal implementation of uasyncio.core (_io_queue) +# Modelled on the asyncio.Stream class (extmod/asyncio/stream.py) +# NOTE: Relies on internal implementation of asyncio.core (_io_queue) class AIOESPNow(espnow.ESPNow): # Read one ESPNow message async def arecv(self): diff --git a/micropython/aiorepl/aiorepl.py b/micropython/aiorepl/aiorepl.py index 14d5d55bc..8f45dfac0 100644 --- a/micropython/aiorepl/aiorepl.py +++ b/micropython/aiorepl/aiorepl.py @@ -41,7 +41,7 @@ async def execute(code, g, s): code = "return {}".format(code) code = """ -import uasyncio as asyncio +import asyncio async def __code(): {} diff --git a/micropython/bluetooth/aioble-central/manifest.py b/micropython/bluetooth/aioble-central/manifest.py index 128c90642..9564ecf77 100644 --- a/micropython/bluetooth/aioble-central/manifest.py +++ b/micropython/bluetooth/aioble-central/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.1") +metadata(version="0.2.2") require("aioble-core") diff --git a/micropython/bluetooth/aioble-core/manifest.py b/micropython/bluetooth/aioble-core/manifest.py index 2448769e6..c2d335b5c 100644 --- a/micropython/bluetooth/aioble-core/manifest.py +++ b/micropython/bluetooth/aioble-core/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.3.0") package( "aioble", diff --git a/micropython/bluetooth/aioble-peripheral/manifest.py b/micropython/bluetooth/aioble-peripheral/manifest.py index dd4dd122d..0aec4d21e 100644 --- a/micropython/bluetooth/aioble-peripheral/manifest.py +++ b/micropython/bluetooth/aioble-peripheral/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.2.1") require("aioble-core") diff --git a/micropython/bluetooth/aioble/aioble/central.py b/micropython/bluetooth/aioble/aioble/central.py index adfc9729e..6d90cd0f8 100644 --- a/micropython/bluetooth/aioble/aioble/central.py +++ b/micropython/bluetooth/aioble/aioble/central.py @@ -6,7 +6,7 @@ import bluetooth import struct -import uasyncio as asyncio +import asyncio from .core import ( ensure_active, @@ -195,12 +195,14 @@ def name(self): # Generator that enumerates the service UUIDs that are advertised. def services(self): - for u in self._decode_field(_ADV_TYPE_UUID16_INCOMPLETE, _ADV_TYPE_UUID16_COMPLETE): - yield bluetooth.UUID(struct.unpack("BBH", _CMD_WRITE_REGISTER, _REG_LSYNCRH, syncword) + self._cmd(">BHH", _CMD_WRITE_REGISTER, _REG_LSYNCRH, syncword) - if "output_power" in lora_cfg: + if not self._configured or any( + key in lora_cfg for key in ("output_power", "pa_ramp_us", "tx_ant") + ): pa_config_args, self._output_power = self._get_pa_tx_params( - lora_cfg["output_power"], lora_cfg.get("tx_ant", None) + lora_cfg.get("output_power", self._output_power), lora_cfg.get("tx_ant", None) ) self._cmd("BBBBB", _CMD_SET_PA_CONFIG, *pa_config_args) - if "pa_ramp_us" in lora_cfg: - self._ramp_val = self._get_pa_ramp_val( - lora_cfg, [10, 20, 40, 80, 200, 800, 1700, 3400] - ) + if "pa_ramp_us" in lora_cfg: + self._ramp_val = self._get_pa_ramp_val( + lora_cfg, [10, 20, 40, 80, 200, 800, 1700, 3400] + ) - if "output_power" in lora_cfg or "pa_ramp_us" in lora_cfg: - # Only send the SetTxParams command if power level or PA ramp time have changed self._cmd("BBB", _CMD_SET_TX_PARAMS, self._output_power, self._ramp_val) - if any(key in lora_cfg for key in ("sf", "bw", "coding_rate")): + if not self._configured or any(key in lora_cfg for key in ("sf", "bw", "coding_rate")): if "sf" in lora_cfg: self._sf = lora_cfg["sf"] if self._sf < _CFG_SF_MIN or self._sf > _CFG_SF_MAX: @@ -441,6 +444,7 @@ def configure(self, lora_cfg): self._reg_write(_REG_RX_GAIN, 0x96 if lora_cfg["rx_boost"] else 0x94) self._check_error() + self._configured = True def _invert_workaround(self, enable): # Apply workaround for DS 15.4 Optimizing the Inverted IQ Operation @@ -465,7 +469,7 @@ def calibrate(self): # See DS 13.1.12 Calibrate Function # calibParam 0xFE means to calibrate all blocks. - self._cmd("BBH", _CMD_SET_RX, timeout >> 16, timeout) + self._cmd(">BBH", _CMD_SET_RX, timeout >> 16, timeout) # 24 bits return self._dio1 @@ -700,11 +704,11 @@ def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None): # have happened well before _cmd() is called again. self._wait_not_busy(self._busy_timeout) - # Pack write_args into _buf and wrap a memoryview of the correct length around it + # Pack write_args into slice of _buf_view memoryview of correct length wrlen = struct.calcsize(fmt) - assert n_read + wrlen <= len(self._buf) # if this fails, make _buf bigger! - struct.pack_into(fmt, self._buf, 0, *write_args) - buf = memoryview(self._buf)[: (wrlen + n_read)] + assert n_read + wrlen <= len(self._buf_view) # if this fails, make _buf bigger! + struct.pack_into(fmt, self._buf_view, 0, *write_args) + buf = self._buf_view[: (wrlen + n_read)] if _DEBUG: print(">>> {}".format(buf[:wrlen].hex())) @@ -719,16 +723,16 @@ def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None): self._cs(1) if n_read > 0: - res = memoryview(buf)[wrlen : (wrlen + n_read)] # noqa: E203 + res = self._buf_view[wrlen : (wrlen + n_read)] # noqa: E203 if _DEBUG: print("<<< {}".format(res.hex())) return res def _reg_read(self, addr): - return self._cmd("BBBB", _CMD_READ_REGISTER, addr >> 8, addr & 0xFF, n_read=1)[0] + return self._cmd(">BHB", _CMD_READ_REGISTER, addr, 0, n_read=1)[0] def _reg_write(self, addr, val): - return self._cmd("BBBB", _CMD_WRITE_REGISTER, addr >> 8, addr & 0xFF, val & 0xFF) + return self._cmd(">BHB", _CMD_WRITE_REGISTER, addr, val & 0xFF) class _SX1262(_SX126x): diff --git a/micropython/lora/lora-sx126x/manifest.py b/micropython/lora/lora-sx126x/manifest.py index 1936a50e4..785a975aa 100644 --- a/micropython/lora/lora-sx126x/manifest.py +++ b/micropython/lora/lora-sx126x/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.1") +metadata(version="0.1.3") require("lora") package("lora") diff --git a/micropython/lora/lora-sx127x/lora/sx127x.py b/micropython/lora/lora-sx127x/lora/sx127x.py index 3f94aaf0c..0226c9696 100644 --- a/micropython/lora/lora-sx127x/lora/sx127x.py +++ b/micropython/lora/lora-sx127x/lora/sx127x.py @@ -519,6 +519,9 @@ def configure(self, lora_cfg): self._reg_update(_REG_MODEM_CONFIG3, update_mask, modem_config3) + if "syncword" in lora_cfg: + self._reg_write(_REG_SYNC_WORD, lora_cfg["syncword"]) + def _reg_write(self, reg, value): self._cs(0) if isinstance(value, int): diff --git a/micropython/lora/lora-sx127x/manifest.py b/micropython/lora/lora-sx127x/manifest.py index 57b9d21d8..1936a50e4 100644 --- a/micropython/lora/lora-sx127x/manifest.py +++ b/micropython/lora/lora-sx127x/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.0") +metadata(version="0.1.1") require("lora") package("lora") diff --git a/micropython/lora/lora-sync/lora/sync_modem.py b/micropython/lora/lora-sync/lora/sync_modem.py index 27c2f19d1..585ae2cb4 100644 --- a/micropython/lora/lora-sync/lora/sync_modem.py +++ b/micropython/lora/lora-sync/lora/sync_modem.py @@ -42,8 +42,8 @@ def send(self, packet, tx_at_ms=None): tx = True while tx is True: - tx = self.poll_send() self._sync_wait(will_irq) + tx = self.poll_send() return tx def recv(self, timeout_ms=None, rx_length=0xFF, rx_packet=None): diff --git a/micropython/lora/lora-sync/manifest.py b/micropython/lora/lora-sync/manifest.py index 57b9d21d8..1936a50e4 100644 --- a/micropython/lora/lora-sync/manifest.py +++ b/micropython/lora/lora-sync/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.0") +metadata(version="0.1.1") require("lora") package("lora") diff --git a/micropython/lora/lora/lora/modem.py b/micropython/lora/lora/lora/modem.py index e71d4ec72..499712acf 100644 --- a/micropython/lora/lora/lora/modem.py +++ b/micropython/lora/lora/lora/modem.py @@ -37,10 +37,11 @@ def __init__(self, ant_sw): self._ant_sw = ant_sw self._irq_callback = None - # Common configuration settings that need to be tracked by all modem drivers - # (Note that subclasses may set these to other values in their constructors, to match - # the power-on-reset configuration of a particular modem.) + # Common configuration settings that need to be tracked by all modem drivers. # + # Where modem hardware sets different values after reset, the driver should + # set them back to these defaults (if not provided by the user), so that + # behaviour remains consistent between different modems using the same driver. self._rf_freq_hz = 0 # Needs to be set via configure() self._sf = 7 # Spreading factor self._bw_hz = 125000 # Reset value diff --git a/micropython/mip/manifest.py b/micropython/mip/manifest.py index 00efa5454..88fb08da1 100644 --- a/micropython/mip/manifest.py +++ b/micropython/mip/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0", description="On-device package installer for network-capable boards") +metadata(version="0.3.0", description="On-device package installer for network-capable boards") require("requests") diff --git a/micropython/mip/mip/__init__.py b/micropython/mip/mip/__init__.py index 68daf32fe..0c3c6f204 100644 --- a/micropython/mip/mip/__init__.py +++ b/micropython/mip/mip/__init__.py @@ -73,6 +73,18 @@ def _rewrite_url(url, branch=None): + "/" + "/".join(url[2:]) ) + elif url.startswith("gitlab:"): + url = url[7:].split("/") + url = ( + "https://gitlab.com/" + + url[0] + + "/" + + url[1] + + "/-/raw/" + + branch + + "/" + + "/".join(url[2:]) + ) return url @@ -128,6 +140,7 @@ def _install_package(package, index, target, version, mpy): package.startswith("http://") or package.startswith("https://") or package.startswith("github:") + or package.startswith("gitlab:") ): if package.endswith(".py") or package.endswith(".mpy"): print("Downloading {} to {}".format(package, target)) diff --git a/micropython/net/ntptime/manifest.py b/micropython/net/ntptime/manifest.py index 97e3c14a3..15f832966 100644 --- a/micropython/net/ntptime/manifest.py +++ b/micropython/net/ntptime/manifest.py @@ -1,3 +1,3 @@ -metadata(description="NTP client.", version="0.1.0") +metadata(description="NTP client.", version="0.1.1") module("ntptime.py", opt=3) diff --git a/micropython/net/ntptime/ntptime.py b/micropython/net/ntptime/ntptime.py index ff0d9d202..d77214d1d 100644 --- a/micropython/net/ntptime/ntptime.py +++ b/micropython/net/ntptime/ntptime.py @@ -1,13 +1,6 @@ -import utime - -try: - import usocket as socket -except: - import socket -try: - import ustruct as struct -except: - import struct +from time import gmtime +import socket +import struct # The NTP host can be configured at runtime by doing: ntptime.host = 'myhost.org' host = "pool.ntp.org" @@ -22,13 +15,38 @@ def time(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: s.settimeout(timeout) - res = s.sendto(NTP_QUERY, addr) + s.sendto(NTP_QUERY, addr) msg = s.recv(48) finally: s.close() val = struct.unpack("!I", msg[40:44])[0] - EPOCH_YEAR = utime.gmtime(0)[0] + # 2024-01-01 00:00:00 converted to an NTP timestamp + MIN_NTP_TIMESTAMP = 3913056000 + + # Y2036 fix + # + # The NTP timestamp has a 32-bit count of seconds, which will wrap back + # to zero on 7 Feb 2036 at 06:28:16. + # + # We know that this software was written during 2024 (or later). + # So we know that timestamps less than MIN_NTP_TIMESTAMP are impossible. + # So if the timestamp is less than MIN_NTP_TIMESTAMP, that probably means + # that the NTP time wrapped at 2^32 seconds. (Or someone set the wrong + # time on their NTP server, but we can't really do anything about that). + # + # So in that case, we need to add in those extra 2^32 seconds, to get the + # correct timestamp. + # + # This means that this code will work until the year 2160. More precisely, + # this code will not work after 7th Feb 2160 at 06:28:15. + # + if val < MIN_NTP_TIMESTAMP: + val += 0x100000000 + + # Convert timestamp from NTP format to our internal format + + EPOCH_YEAR = gmtime(0)[0] if EPOCH_YEAR == 2000: # (date(2000, 1, 1) - date(1900, 1, 1)).days * 24*60*60 NTP_DELTA = 3155673600 @@ -46,5 +64,5 @@ def settime(): t = time() import machine - tm = utime.gmtime(t) + tm = gmtime(t) machine.RTC().datetime((tm[0], tm[1], tm[2], tm[6] + 1, tm[3], tm[4], tm[5], 0)) diff --git a/micropython/senml/examples/basic_cbor.py b/micropython/senml/examples/basic_cbor.py index f5e92af37..804a886fc 100644 --- a/micropython/senml/examples/basic_cbor.py +++ b/micropython/senml/examples/basic_cbor.py @@ -26,7 +26,7 @@ from senml import * import time -from cbor2 import decoder +import cbor2 pack = SenmlPack("device_name") @@ -38,5 +38,5 @@ cbor_val = pack.to_cbor() print(cbor_val) print(cbor_val.hex()) - print(decoder.loads(cbor_val)) # convert to string again so we can print it. + print(cbor2.loads(cbor_val)) # convert to string again so we can print it. time.sleep(1) diff --git a/micropython/senml/manifest.py b/micropython/senml/manifest.py index 216717caf..f4743075a 100644 --- a/micropython/senml/manifest.py +++ b/micropython/senml/manifest.py @@ -1,6 +1,6 @@ metadata( description="SenML serialisation for MicroPython.", - version="0.1.0", + version="0.1.1", pypi_publish="micropython-senml", ) diff --git a/micropython/senml/senml/senml_pack.py b/micropython/senml/senml/senml_pack.py index 85b26d40b..4e106fd3e 100644 --- a/micropython/senml/senml/senml_pack.py +++ b/micropython/senml/senml/senml_pack.py @@ -27,8 +27,7 @@ from senml.senml_record import SenmlRecord from senml.senml_base import SenmlBase import json -from cbor2 import encoder -from cbor2 import decoder +import cbor2 class SenmlPackIterator: @@ -278,7 +277,7 @@ def from_cbor(self, data): :param data: a byte array. :return: None """ - records = decoder.loads(data) # load the raw senml data + records = cbor2.loads(data) # load the raw senml data naming_map = { "bn": -2, "bt": -3, @@ -320,7 +319,7 @@ def to_cbor(self): } converted = [] self._build_rec_dict(naming_map, converted) - return encoder.dumps(converted) + return cbor2.dumps(converted) def add(self, item): """ diff --git a/micropython/uaiohttpclient/README b/micropython/uaiohttpclient/README index a3d88b0a4..1222f9d61 100644 --- a/micropython/uaiohttpclient/README +++ b/micropython/uaiohttpclient/README @@ -1,4 +1,4 @@ -uaiohttpclient is an HTTP client module for MicroPython uasyncio module, +uaiohttpclient is an HTTP client module for MicroPython asyncio module, with API roughly compatible with aiohttp (https://github.com/KeepSafe/aiohttp) module. Note that only client is implemented, for server see picoweb microframework. diff --git a/micropython/uaiohttpclient/example.py b/micropython/uaiohttpclient/example.py index d265c9db7..540d1b3de 100644 --- a/micropython/uaiohttpclient/example.py +++ b/micropython/uaiohttpclient/example.py @@ -2,7 +2,7 @@ # uaiohttpclient - fetch URL passed as command line argument. # import sys -import uasyncio as asyncio +import asyncio import uaiohttpclient as aiohttp diff --git a/micropython/uaiohttpclient/manifest.py b/micropython/uaiohttpclient/manifest.py index a204d57b2..8b35e0a70 100644 --- a/micropython/uaiohttpclient/manifest.py +++ b/micropython/uaiohttpclient/manifest.py @@ -1,4 +1,4 @@ -metadata(description="HTTP client module for MicroPython uasyncio module", version="0.5.2") +metadata(description="HTTP client module for MicroPython asyncio module", version="0.5.2") # Originally written by Paul Sokolovsky. diff --git a/micropython/uaiohttpclient/uaiohttpclient.py b/micropython/uaiohttpclient/uaiohttpclient.py index 6347c3371..2e782638c 100644 --- a/micropython/uaiohttpclient/uaiohttpclient.py +++ b/micropython/uaiohttpclient/uaiohttpclient.py @@ -1,4 +1,4 @@ -import uasyncio as asyncio +import asyncio class ClientResponse: diff --git a/micropython/ucontextlib/tests.py b/micropython/ucontextlib/tests.py index 4fd026ae7..163175d82 100644 --- a/micropython/ucontextlib/tests.py +++ b/micropython/ucontextlib/tests.py @@ -24,7 +24,7 @@ def test_context_manager(self): def test_context_manager_on_error(self): exc = Exception() try: - with self._manager(123) as x: + with self._manager(123): raise exc except Exception as e: self.assertEqual(exc, e) diff --git a/micropython/udnspkt/example_resolve.py b/micropython/udnspkt/example_resolve.py index c1215045a..d72c17a48 100644 --- a/micropython/udnspkt/example_resolve.py +++ b/micropython/udnspkt/example_resolve.py @@ -1,15 +1,15 @@ -import uio -import usocket +import io +import socket import udnspkt -s = usocket.socket(usocket.AF_INET, usocket.SOCK_DGRAM) -dns_addr = usocket.getaddrinfo("127.0.0.1", 53)[0][-1] +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +dns_addr = socket.getaddrinfo("127.0.0.1", 53)[0][-1] def resolve(domain, is_ipv6): - buf = uio.BytesIO(48) + buf = io.BytesIO(48) udnspkt.make_req(buf, "google.com", is_ipv6) v = buf.getvalue() print("query: ", v) @@ -17,11 +17,11 @@ def resolve(domain, is_ipv6): resp = s.recv(1024) print("resp:", resp) - buf = uio.BytesIO(resp) + buf = io.BytesIO(resp) addr = udnspkt.parse_resp(buf, is_ipv6) print("bin addr:", addr) - print("addr:", usocket.inet_ntop(usocket.AF_INET6 if is_ipv6 else usocket.AF_INET, addr)) + print("addr:", socket.inet_ntop(socket.AF_INET6 if is_ipv6 else socket.AF_INET, addr)) resolve("google.com", False) diff --git a/micropython/udnspkt/udnspkt.py b/micropython/udnspkt/udnspkt.py index e55285975..f3b998a8a 100644 --- a/micropython/udnspkt/udnspkt.py +++ b/micropython/udnspkt/udnspkt.py @@ -1,6 +1,3 @@ -import uio - - def write_fqdn(buf, name): parts = name.split(".") for p in parts: @@ -43,36 +40,28 @@ def parse_resp(buf, is_ipv6): if is_ipv6: typ = 28 # AAAA - id = buf.readbin(">H") + buf.readbin(">H") # id flags = buf.readbin(">H") assert flags & 0x8000 - qcnt = buf.readbin(">H") + buf.readbin(">H") # qcnt acnt = buf.readbin(">H") - nscnt = buf.readbin(">H") - addcnt = buf.readbin(">H") - # print(qcnt, acnt, nscnt, addcnt) + buf.readbin(">H") # nscnt + buf.readbin(">H") # addcnt skip_fqdn(buf) - v = buf.readbin(">H") - # print(v) - v = buf.readbin(">H") - # print(v) + buf.readbin(">H") + buf.readbin(">H") for i in range(acnt): # print("Resp #%d" % i) # v = read_fqdn(buf) # print(v) skip_fqdn(buf) - t = buf.readbin(">H") - # print("Type", t) - v = buf.readbin(">H") - # print("Class", v) - v = buf.readbin(">I") - # print("TTL", v) + t = buf.readbin(">H") # Type + buf.readbin(">H") # Class + buf.readbin(">I") # TTL rlen = buf.readbin(">H") - # print("rlen", rlen) rval = buf.read(rlen) - # print(rval) if t == typ: return rval diff --git a/micropython/umqtt.robust/example_sub_robust.py b/micropython/umqtt.robust/example_sub_robust.py index c991c70a1..f09befe02 100644 --- a/micropython/umqtt.robust/example_sub_robust.py +++ b/micropython/umqtt.robust/example_sub_robust.py @@ -19,8 +19,7 @@ def sub_cb(topic, msg): # # There can be a problem when a session for a given client exists, # but doesn't have subscriptions a particular application expects. -# In this case, a session needs to be cleaned first. See -# example_reset_session.py for an obvious way how to do that. +# In this case, a session needs to be cleaned first. # # In an actual application, it's up to its developer how to # manage these issues. One extreme is to have external "provisioning" diff --git a/micropython/umqtt.robust/umqtt/robust.py b/micropython/umqtt.robust/umqtt/robust.py index 4cc10e336..51596de9e 100644 --- a/micropython/umqtt.robust/umqtt/robust.py +++ b/micropython/umqtt.robust/umqtt/robust.py @@ -1,4 +1,4 @@ -import utime +import time from . import simple @@ -7,7 +7,7 @@ class MQTTClient(simple.MQTTClient): DEBUG = False def delay(self, i): - utime.sleep(self.DELAY) + time.sleep(self.DELAY) def log(self, in_reconnect, e): if self.DEBUG: diff --git a/micropython/umqtt.simple/example_pub_button.py b/micropython/umqtt.simple/example_pub_button.py index 1bc47bc5e..2a3ec851e 100644 --- a/micropython/umqtt.simple/example_pub_button.py +++ b/micropython/umqtt.simple/example_pub_button.py @@ -1,5 +1,5 @@ import time -import ubinascii +import binascii import machine from umqtt.simple import MQTTClient from machine import Pin @@ -10,7 +10,7 @@ # Default MQTT server to connect to SERVER = "192.168.1.35" -CLIENT_ID = ubinascii.hexlify(machine.unique_id()) +CLIENT_ID = binascii.hexlify(machine.unique_id()) TOPIC = b"led" diff --git a/micropython/umqtt.simple/example_sub_led.py b/micropython/umqtt.simple/example_sub_led.py index 73c6b58d8..c3dcf08d2 100644 --- a/micropython/umqtt.simple/example_sub_led.py +++ b/micropython/umqtt.simple/example_sub_led.py @@ -1,6 +1,6 @@ from umqtt.simple import MQTTClient from machine import Pin -import ubinascii +import binascii import machine import micropython @@ -11,7 +11,7 @@ # Default MQTT server to connect to SERVER = "192.168.1.35" -CLIENT_ID = ubinascii.hexlify(machine.unique_id()) +CLIENT_ID = binascii.hexlify(machine.unique_id()) TOPIC = b"led" diff --git a/micropython/umqtt.simple/manifest.py b/micropython/umqtt.simple/manifest.py index 19617a5ee..b418995c5 100644 --- a/micropython/umqtt.simple/manifest.py +++ b/micropython/umqtt.simple/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Lightweight MQTT client for MicroPython.", version="1.3.4") +metadata(description="Lightweight MQTT client for MicroPython.", version="1.4.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index 2b269473b..6da38e445 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -1,6 +1,6 @@ -import usocket as socket -import ustruct as struct -from ubinascii import hexlify +import socket +import struct +from binascii import hexlify class MQTTException(Exception): @@ -16,8 +16,7 @@ def __init__( user=None, password=None, keepalive=0, - ssl=False, - ssl_params={}, + ssl=None, ): if port == 0: port = 8883 if ssl else 1883 @@ -26,7 +25,6 @@ def __init__( self.server = server self.port = port self.ssl = ssl - self.ssl_params = ssl_params self.pid = 0 self.cb = None self.user = user @@ -67,15 +65,13 @@ def connect(self, clean_session=True): addr = socket.getaddrinfo(self.server, self.port)[0][-1] self.sock.connect(addr) if self.ssl: - import ussl - - self.sock = ussl.wrap_socket(self.sock, **self.ssl_params) + self.sock = self.ssl.wrap_socket(self.sock, server_hostname=self.server) premsg = bytearray(b"\x10\0\0\0\0\0") msg = bytearray(b"\x04MQTT\x04\x02\0\0") sz = 10 + 2 + len(self.client_id) msg[6] = clean_session << 1 - if self.user is not None: + if self.user: sz += 2 + len(self.user) + 2 + len(self.pswd) msg[6] |= 0xC0 if self.keepalive: @@ -101,7 +97,7 @@ def connect(self, clean_session=True): if self.lw_topic: self._send_str(self.lw_topic) self._send_str(self.lw_msg) - if self.user is not None: + if self.user: self._send_str(self.user) self._send_str(self.pswd) resp = self.sock.read(4) diff --git a/micropython/urllib.urequest/manifest.py b/micropython/urllib.urequest/manifest.py index e3e8f13f4..2790208a7 100644 --- a/micropython/urllib.urequest/manifest.py +++ b/micropython/urllib.urequest/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.6.0") +metadata(version="0.7.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/urllib.urequest/urllib/urequest.py b/micropython/urllib.urequest/urllib/urequest.py index 4c654d45e..f83cbaa94 100644 --- a/micropython/urllib.urequest/urllib/urequest.py +++ b/micropython/urllib.urequest/urllib/urequest.py @@ -1,4 +1,4 @@ -import usocket +import socket def urlopen(url, data=None, method="GET"): @@ -12,7 +12,7 @@ def urlopen(url, data=None, method="GET"): if proto == "http:": port = 80 elif proto == "https:": - import ussl + import tls port = 443 else: @@ -22,14 +22,16 @@ def urlopen(url, data=None, method="GET"): host, port = host.split(":", 1) port = int(port) - ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] - s = usocket.socket(ai[0], ai[1], ai[2]) + s = socket.socket(ai[0], ai[1], ai[2]) try: s.connect(ai[-1]) if proto == "https:": - s = ussl.wrap_socket(s, server_hostname=host) + context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + context.verify_mode = tls.CERT_NONE + s = context.wrap_socket(s, server_hostname=host) s.write(method) s.write(b" /") @@ -46,10 +48,10 @@ def urlopen(url, data=None, method="GET"): if data: s.write(data) - l = s.readline() - l = l.split(None, 2) + l = s.readline() # Status-Line + # l = l.split(None, 2) # print(l) - status = int(l[1]) + # status = int(l[1]) # FIXME: Status-Code element is not currently checked while True: l = s.readline() if not l or l == b"\r\n": diff --git a/micropython/usb/README.md b/micropython/usb/README.md new file mode 100644 index 000000000..d4b975d12 --- /dev/null +++ b/micropython/usb/README.md @@ -0,0 +1,148 @@ +# USB Drivers + +These packages allow implementing USB functionality on a MicroPython system +using pure Python code. + +Currently only USB device is implemented, not USB host. + +## USB Device support + +### Support + +USB Device support depends on the low-level +[machine.USBDevice](https://docs.micropython.org/en/latest/library/machine.USBDevice.html) +class. This class is new and not supported on all ports, so please check the +documentation for your MicroPython version. It is possible to implement a USB +device using only the low-level USBDevice class. However, the packages here are +higher level and easier to use. + +For more information about how to install packages, or "freeze" them into a +firmware image, consult the [MicroPython documentation on "Package +management"](https://docs.micropython.org/en/latest/reference/packages.html). + +### Examples + +The [examples/device](examples/device) directory in this repo has a range of +examples. After installing necessary packages, you can download an example and +run it with `mpremote run EXAMPLE_FILENAME.py` ([mpremote +docs](https://docs.micropython.org/en/latest/reference/mpremote.html#mpremote-command-run)). + +#### Unexpected serial disconnects + +If you normally connect to your MicroPython device over a USB serial port ("USB +CDC"), then running a USB example will disconnect mpremote when the new USB +device configuration activates and the serial port has to temporarily +disconnect. It is likely that mpremote will print an error. The example should +still start running, if necessary then you can reconnect with mpremote and type +Ctrl-B to restore the MicroPython REPL and/or Ctrl-C to stop the running +example. + +If you use `mpremote run` again while a different USB device configuration is +already active, then the USB serial port may disconnect immediately before the +example runs. This is because mpremote has to soft-reset MicroPython, and when +the existing USB device is reset then the entire USB port needs to reset. If +this happens, run the same `mpremote run` command again. + +We plan to add features to `mpremote` so that this limitation is less +disruptive. Other tools that communicate with MicroPython over the serial port +will encounter similar issues when runtime USB is in use. + +### Initialising runtime USB + +The overall pattern for enabling USB devices at runtime is: + +1. Instantiate the Interface objects for your desired USB device. +2. Call `usb.device.get()` to get the singleton object for the high-level USB device. +3. Call `init(...)` to pass the desired interfaces as arguments, plus any custom + keyword arguments to configure the overall device. + +An example, similar to [mouse_example.py](examples/device/mouse_example.py): + +```py + m = usb.device.mouse.MouseInterface() + usb.device.get().init(m, builtin_driver=True) +``` + +Setting `builtin_driver=True` means that any built-in USB serial port will still +be available. Otherwise, you may permanently lose access to MicroPython until +the next time the device resets. + +See [Unexpected serial disconnects](#Unexpected-serial-disconnects), above, for +an explanation of possible errors or disconnects when the runtime USB device +initialises. + +Placing the call to `usb.device.get().init()` into the `boot.py` of the +MicroPython file system allows the runtime USB device to initialise immediately +on boot, before any built-in USB. This is a feature (not a bug) and allows you +full control over the USB device, for example to only enable USB HID and prevent +REPL access to the system. + +However, note that calling this function on boot without `builtin_driver=True` +will make the MicroPython USB serial interface permanently inaccessible until +you "safe mode boot" (on supported boards) or completely erase the flash of your +device. + +### Package usb-device + +This base package contains the common implementation components for the other +packages, and can be used to implement new and different USB interface support. +All of the other `usb-device-` packages depend on this package, and it +will be automatically installed as needed. + +Specicially, this package provides the `usb.device.get()` function for accessing +the Device singleton object, and the `usb.device.core` module which contains the +low-level classes and utility functions for implementing new USB interface +drivers in Python. The best examples of how to use the core classes is the +source code of the other USB device packages. + +### Package usb-device-keyboard + +This package provides the `usb.device.keyboard` module. See +[keyboard_example.py](examples/device/keyboard_example.py) for an example +program. + +### Package usb-device-mouse + +This package provides the `usb.device.mouse` module. See +[mouse_example.py](examples/device/mouse_example.py) for an example program. + +### Package usb-device-hid + +This package provides the `usb.device.hid` module. USB HID (Human Interface +Device) class allows creating a wide variety of device types. The most common +are mouse and keyboard, which have their own packages in micropython-lib. +However, using the usb-device-hid package directly allows creation of any kind +of HID device. + +See [hid_custom_keypad_example.py](examples/device/hid_custom_keypad_example.py) +for an example of a Keypad HID device with a custom HID descriptor. + +### Package usb-device-cdc + +This package provides the `usb.device.cdc` module. USB CDC (Communications +Device Class) is most commonly used for virtual serial port USB interfaces, and +that is what is supported here. + +The example [cdc_repl_example.py](examples/device/cdc_repl_example.py) +demonstrates how to add a second USB serial interface and duplicate the +MicroPython REPL between the two. + +### Package usb-device-midi + +This package provides the `usb.device.midi` module. This allows implementing +USB MIDI devices in MicroPython. + +The example [midi_example.py](examples/device/midi_example.py) demonstrates how +to create a simple MIDI device to send MIDI data to and from the USB host. + +### Limitations + +#### Buffer thread safety + +The internal Buffer class that's used by most of the USB device classes expects data +to be written to it (i.e. sent to the host) by only one thread. Bytes may be +lost from the USB transfers if more than one thread (or a thread and a callback) +try to write to the buffer simultaneously. + +If writing USB data from multiple sources, your code may need to add +synchronisation (i.e. locks). diff --git a/micropython/usb/examples/device/cdc_repl_example.py b/micropython/usb/examples/device/cdc_repl_example.py new file mode 100644 index 000000000..06dc9a76c --- /dev/null +++ b/micropython/usb/examples/device/cdc_repl_example.py @@ -0,0 +1,47 @@ +# MicroPython USB CDC REPL example +# +# Example demonstrating how to use os.dupterm() to provide the +# MicroPython REPL on a dynamic CDCInterface() serial port. +# +# To run this example: +# +# 1. Make sure `usb-device-cdc` is installed via: mpremote mip install usb-device-cdc +# +# 2. Run the example via: mpremote run cdc_repl_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the second serial port. If you check (for example by running mpremote connect +# list) then you should now see two USB serial devices. +# +# 4. Connect to one of the new ports: mpremote connect PORTNAME +# +# It may be necessary to type Ctrl-B to exit the raw REPL mode and resume the +# interactive REPL after mpremote connects. +# +# MIT license; Copyright (c) 2023-2024 Angus Gratton +import os +import time +import usb.device +from usb.device.cdc import CDCInterface + +cdc = CDCInterface() +cdc.init(timeout=0) # zero timeout makes this non-blocking, suitable for os.dupterm() + +# pass builtin_driver=True so that we get the built-in USB-CDC alongside, +# if it's available. +usb.device.get().init(cdc, builtin_driver=True) + +print("Waiting for USB host to configure the interface...") + +# wait for host enumerate as a CDC device... +while not cdc.is_open(): + time.sleep_ms(100) + +# Note: This example doesn't wait for the host to access the new CDC port, +# which could be done by polling cdc.dtr, as this will block the REPL +# from resuming while this code is still executing. + +print("CDC port enumerated, duplicating REPL...") + +old_term = os.dupterm(cdc) diff --git a/micropython/usb/examples/device/hid_custom_keypad_example.py b/micropython/usb/examples/device/hid_custom_keypad_example.py new file mode 100644 index 000000000..9d427cf10 --- /dev/null +++ b/micropython/usb/examples/device/hid_custom_keypad_example.py @@ -0,0 +1,144 @@ +# MicroPython USB HID custom Keypad example +# +# This example demonstrates creating a custom HID device with its own +# HID descriptor, in this case for a USB number keypad. +# +# For higher level examples that require less code to use, see mouse_example.py +# and keyboard_example.py +# +# To run this example: +# +# 1. Make sure `usb-device-hid` is installed via: mpremote mip install usb-device-hid +# +# 2. Run the example via: mpremote run hid_custom_keypad_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the custom HID interface present. At this point, the example is running. +# +# 4. To see output from the example, re-connect: mpremote connect PORTNAME +# +# MIT license; Copyright (c) 2023 Dave Wickham, 2023-2024 Angus Gratton +from micropython import const +import time +import usb.device +from usb.device.hid import HIDInterface + +_INTERFACE_PROTOCOL_KEYBOARD = const(0x01) + + +def keypad_example(): + k = KeypadInterface() + + usb.device.get().init(k, builtin_driver=True) + + while not k.is_open(): + time.sleep_ms(100) + + while True: + time.sleep(2) + print("Press NumLock...") + k.send_key("") + time.sleep_ms(100) + k.send_key() + time.sleep(1) + # continue + print("Press ...") + for _ in range(3): + time.sleep(0.1) + k.send_key(".") + time.sleep(0.1) + k.send_key() + print("Starting again...") + + +class KeypadInterface(HIDInterface): + # Very basic synchronous USB keypad HID interface + + def __init__(self): + super().__init__( + _KEYPAD_REPORT_DESC, + set_report_buf=bytearray(1), + protocol=_INTERFACE_PROTOCOL_KEYBOARD, + interface_str="MicroPython Keypad", + ) + self.numlock = False + + def on_set_report(self, report_data, _report_id, _report_type): + report = report_data[0] + b = bool(report & 1) + if b != self.numlock: + print("Numlock: ", b) + self.numlock = b + + def send_key(self, key=None): + if key is None: + self.send_report(b"\x00") + else: + self.send_report(_key_to_id(key).to_bytes(1, "big")) + + +# See HID Usages and Descriptions 1.4, section 10 Keyboard/Keypad Page (0x07) +# +# This keypad example has a contiguous series of keys (KEYPAD_KEY_IDS) starting +# from the NumLock/Clear keypad key (0x53), but you can send any Key IDs from +# the table in the HID Usages specification. +_KEYPAD_KEY_OFFS = const(0x53) + +_KEYPAD_KEY_IDS = [ + "", + "/", + "*", + "-", + "+", + "", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + ".", +] + + +def _key_to_id(key): + # This is a little slower than making a dict for lookup, but uses + # less memory and O(n) can be fast enough when n is small. + return _KEYPAD_KEY_IDS.index(key) + _KEYPAD_KEY_OFFS + + +# HID Report descriptor for a numeric keypad +# +# fmt: off +_KEYPAD_REPORT_DESC = ( + b'\x05\x01' # Usage Page (Generic Desktop) + b'\x09\x07' # Usage (Keypad) + b'\xA1\x01' # Collection (Application) + b'\x05\x07' # Usage Page (Keypad) + b'\x19\x00' # Usage Minimum (0) + b'\x29\xFF' # Usage Maximum (ff) + b'\x15\x00' # Logical Minimum (0) + b'\x25\xFF' # Logical Maximum (ff) + b'\x95\x01' # Report Count (1), + b'\x75\x08' # Report Size (8), + b'\x81\x00' # Input (Data, Array, Absolute) + b'\x05\x08' # Usage page (LEDs) + b'\x19\x01' # Usage Minimum (1) + b'\x29\x01' # Usage Maximum (1), + b'\x95\x01' # Report Count (1), + b'\x75\x01' # Report Size (1), + b'\x91\x02' # Output (Data, Variable, Absolute) + b'\x95\x01' # Report Count (1), + b'\x75\x07' # Report Size (7), + b'\x91\x01' # Output (Constant) - padding bits + b'\xC0' # End Collection +) +# fmt: on + + +keypad_example() diff --git a/micropython/usb/examples/device/keyboard_example.py b/micropython/usb/examples/device/keyboard_example.py new file mode 100644 index 000000000..d8994ff1b --- /dev/null +++ b/micropython/usb/examples/device/keyboard_example.py @@ -0,0 +1,97 @@ +# MicroPython USB Keyboard example +# +# To run this example: +# +# 1. Check the KEYS assignment below, and connect buttons or switches to the +# assigned GPIOs. You can change the entries as needed, look up the reference +# for your board to see what pins are available. Note that the example uses +# "active low" logic, so pressing a switch or button should switch the +# connected pin to Ground (0V). +# +# 2. Make sure `usb-device-keyboard` is installed via: mpremote mip install usb-device-keyboard +# +# 3. Run the example via: mpremote run keyboard_example.py +# +# 4. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the keyboard interface present. At this point, the example is running. +# +# 5. The example doesn't print anything to the serial port, but to stop it first +# re-connect: mpremote connect PORTNAME +# +# 6. Type Ctrl-C to interrupt the running example and stop it. You may have to +# also type Ctrl-B to restore the interactive REPL. +# +# To implement a keyboard with different USB HID characteristics, copy the +# usb-device-keyboard/usb/device/keyboard.py file into your own project and modify +# KeyboardInterface. +# +# MIT license; Copyright (c) 2024 Angus Gratton +import usb.device +from usb.device.keyboard import KeyboardInterface, KeyCode, LEDCode +from machine import Pin +import time + +# Tuples mapping Pin inputs to the KeyCode each input generates +# +# (Big keyboards usually multiplex multiple keys per input with a scan matrix, +# but this is a simple example.) +KEYS = ( + (Pin.cpu.GPIO10, KeyCode.CAPS_LOCK), + (Pin.cpu.GPIO11, KeyCode.LEFT_SHIFT), + (Pin.cpu.GPIO12, KeyCode.M), + (Pin.cpu.GPIO13, KeyCode.P), + # ... add more pin to KeyCode mappings here if needed +) + +# Tuples mapping Pin outputs to the LEDCode that turns the output on +LEDS = ( + (Pin.board.LED, LEDCode.CAPS_LOCK), + # ... add more pin to LEDCode mappings here if needed +) + + +class ExampleKeyboard(KeyboardInterface): + def on_led_update(self, led_mask): + # print(hex(led_mask)) + for pin, code in LEDS: + # Set the pin high if 'code' bit is set in led_mask + pin(code & led_mask) + + +def keyboard_example(): + # Initialise all the pins as active-low inputs with pullup resistors + for pin, _ in KEYS: + pin.init(Pin.IN, Pin.PULL_UP) + + # Initialise all the LEDs as active-high outputs + for pin, _ in LEDS: + pin.init(Pin.OUT, value=0) + + # Register the keyboard interface and re-enumerate + k = ExampleKeyboard() + usb.device.get().init(k, builtin_driver=True) + + print("Entering keyboard loop...") + + keys = [] # Keys held down, reuse the same list object + prev_keys = [None] # Previous keys, starts with a dummy value so first + # iteration will always send + while True: + if k.is_open(): + keys.clear() + for pin, code in KEYS: + if not pin(): # active-low + keys.append(code) + if keys != prev_keys: + # print(keys) + k.send_keys(keys) + prev_keys.clear() + prev_keys.extend(keys) + + # This simple example scans each input in an infinite loop, but a more + # complex implementation would probably use a timer or similar. + time.sleep_ms(1) + + +keyboard_example() diff --git a/micropython/usb/examples/device/midi_example.py b/micropython/usb/examples/device/midi_example.py new file mode 100644 index 000000000..55fe8af69 --- /dev/null +++ b/micropython/usb/examples/device/midi_example.py @@ -0,0 +1,78 @@ +# MicroPython USB MIDI example +# +# This example demonstrates creating a custom MIDI device. +# +# To run this example: +# +# 1. Make sure `usb-device-midi` is installed via: mpremote mip install usb-device-midi +# +# 2. Run the example via: mpremote run midi_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the MIDI interface present. At this point, the example is running. +# +# 4. To see output from the example, re-connect: mpremote connect PORTNAME +# +# +# MIT license; Copyright (c) 2023-2024 Angus Gratton +import usb.device +from usb.device.midi import MIDIInterface +import time + + +class MIDIExample(MIDIInterface): + # Very simple example event handler functions, showing how to receive note + # and control change messages sent from the host to the device. + # + # If you need to send MIDI data to the host, then it's fine to instantiate + # MIDIInterface class directly. + + def on_open(self): + super().on_open() + print("Device opened by host") + + def on_note_on(self, channel, pitch, vel): + print(f"RX Note On channel {channel} pitch {pitch} velocity {vel}") + + def on_note_off(self, channel, pitch, vel): + print(f"RX Note Off channel {channel} pitch {pitch} velocity {vel}") + + def on_control_change(self, channel, controller, value): + print(f"RX Control channel {channel} controller {controller} value {value}") + + +m = MIDIExample() +# Remove builtin_driver=True if you don't want the MicroPython serial REPL available. +usb.device.get().init(m, builtin_driver=True) + +print("Waiting for USB host to configure the interface...") + +while not m.is_open(): + time.sleep_ms(100) + +print("Starting MIDI loop...") + +# TX constants +CHANNEL = 0 +PITCH = 60 +CONTROLLER = 64 + +control_val = 0 + +while m.is_open(): + time.sleep(1) + print(f"TX Note On channel {CHANNEL} pitch {PITCH}") + m.note_on(CHANNEL, PITCH) # Velocity is an optional third argument + time.sleep(0.5) + print(f"TX Note Off channel {CHANNEL} pitch {PITCH}") + m.note_off(CHANNEL, PITCH) + time.sleep(1) + print(f"TX Control channel {CHANNEL} controller {CONTROLLER} value {control_val}") + m.control_change(CHANNEL, CONTROLLER, control_val) + control_val += 1 + if control_val == 0x7F: + control_val = 0 + time.sleep(1) + +print("USB host has reset device, example done.") diff --git a/micropython/usb/examples/device/mouse_example.py b/micropython/usb/examples/device/mouse_example.py new file mode 100644 index 000000000..c73d6cfa6 --- /dev/null +++ b/micropython/usb/examples/device/mouse_example.py @@ -0,0 +1,52 @@ +# MicroPython USB Mouse example +# +# To run this example: +# +# 1. Make sure `usb-device-mouse` is installed via: mpremote mip install usb-device-mouse +# +# 2. Run the example via: mpremote run mouse_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the mouse interface present. At this point, the example is running. +# +# 4. You should see the mouse move and right click. At this point, the example +# is finished executing. +# +# To implement a more complex mouse with more buttons or other custom interface +# features, copy the usb-device-mouse/usb/device/mouse.py file into your own +# project and modify MouseInterface. +# +# MIT license; Copyright (c) 2023-2024 Angus Gratton +import time +import usb.device +from usb.device.mouse import MouseInterface + + +def mouse_example(): + m = MouseInterface() + + # Note: builtin_driver=True means that if there's a USB-CDC REPL + # available then it will appear as well as the HID device. + usb.device.get().init(m, builtin_driver=True) + + # wait for host to enumerate as a HID device... + while not m.is_open(): + time.sleep_ms(100) + + time.sleep_ms(2000) + + print("Moving...") + m.move_by(-100, 0) + m.move_by(-100, 0) + time.sleep_ms(500) + + print("Clicking...") + m.click_right(True) + time.sleep_ms(200) + m.click_right(False) + + print("Done!") + + +mouse_example() diff --git a/micropython/usb/usb-device-cdc/manifest.py b/micropython/usb/usb-device-cdc/manifest.py new file mode 100644 index 000000000..4520325e3 --- /dev/null +++ b/micropython/usb/usb-device-cdc/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.1") +require("usb-device") +package("usb") diff --git a/micropython/usb/usb-device-cdc/usb/device/cdc.py b/micropython/usb/usb-device-cdc/usb/device/cdc.py new file mode 100644 index 000000000..28bfb0657 --- /dev/null +++ b/micropython/usb/usb-device-cdc/usb/device/cdc.py @@ -0,0 +1,441 @@ +# MicroPython USB CDC module +# MIT license; Copyright (c) 2022 Martin Fischer, 2023-2024 Angus Gratton +import io +import time +import errno +import machine +import struct +from micropython import const + +from .core import Interface, Buffer, split_bmRequestType + +_EP_IN_FLAG = const(1 << 7) + +# Control transfer stages +_STAGE_IDLE = const(0) +_STAGE_SETUP = const(1) +_STAGE_DATA = const(2) +_STAGE_ACK = const(3) + +# Request types +_REQ_TYPE_STANDARD = const(0x0) +_REQ_TYPE_CLASS = const(0x1) +_REQ_TYPE_VENDOR = const(0x2) +_REQ_TYPE_RESERVED = const(0x3) + +_DEV_CLASS_MISC = const(0xEF) +_CS_DESC_TYPE = const(0x24) # CS Interface type communication descriptor + +# CDC control interface definitions +_INTERFACE_CLASS_CDC = const(2) +_INTERFACE_SUBCLASS_CDC = const(2) # Abstract Control Mode +_PROTOCOL_NONE = const(0) # no protocol + +# CDC descriptor subtype +# see also CDC120.pdf, table 13 +_CDC_FUNC_DESC_HEADER = const(0) +_CDC_FUNC_DESC_CALL_MANAGEMENT = const(1) +_CDC_FUNC_DESC_ABSTRACT_CONTROL = const(2) +_CDC_FUNC_DESC_UNION = const(6) + +# CDC class requests, table 13, PSTN subclass +_SET_LINE_CODING_REQ = const(0x20) +_GET_LINE_CODING_REQ = const(0x21) +_SET_CONTROL_LINE_STATE = const(0x22) +_SEND_BREAK_REQ = const(0x23) + +_LINE_CODING_STOP_BIT_1 = const(0) +_LINE_CODING_STOP_BIT_1_5 = const(1) +_LINE_CODING_STOP_BIT_2 = const(2) + +_LINE_CODING_PARITY_NONE = const(0) +_LINE_CODING_PARITY_ODD = const(1) +_LINE_CODING_PARITY_EVEN = const(2) +_LINE_CODING_PARITY_MARK = const(3) +_LINE_CODING_PARITY_SPACE = const(4) + +_LINE_STATE_DTR = const(1) +_LINE_STATE_RTS = const(2) + +_PARITY_BITS_REPR = "NOEMS" +_STOP_BITS_REPR = ("1", "1.5", "2") + +# Other definitions +_CDC_VERSION = const(0x0120) # release number in binary-coded decimal + +# Number of endpoints in each interface +_CDC_CONTROL_EP_NUM = const(1) +_CDC_DATA_EP_NUM = const(2) + +# CDC data interface definitions +_CDC_ITF_DATA_CLASS = const(0xA) +_CDC_ITF_DATA_SUBCLASS = const(0) +_CDC_ITF_DATA_PROT = const(0) # no protocol + +# Length of the bulk transfer endpoints. Maybe should be configurable? +_BULK_EP_LEN = const(64) + +# MicroPython error constants (negated as IOBase.ioctl uses negative return values for error codes) +# these must match values in py/mperrno.h +_MP_EINVAL = const(-22) +_MP_ETIMEDOUT = const(-110) + +# MicroPython stream ioctl requests, same as py/stream.h +_MP_STREAM_FLUSH = const(1) +_MP_STREAM_POLL = const(3) + +# MicroPython ioctl poll values, same as py/stream.h +_MP_STREAM_POLL_WR = const(0x04) +_MP_STREAM_POLL_RD = const(0x01) +_MP_STREAM_POLL_HUP = const(0x10) + + +class CDCInterface(io.IOBase, Interface): + # USB CDC serial device class, designed to resemble machine.UART + # with some additional methods. + # + # Relies on multiple inheritance so it can be an io.IOBase for stream + # functions and also a Interface (actually an Interface Association + # Descriptor holding two interfaces.) + def __init__(self, **kwargs): + # io.IOBase has no __init__() + Interface.__init__(self) + + # Callbacks for particular control changes initiated by the host + self.break_cb = None # Host sent a "break" condition + self.line_state_cb = None + self.line_coding_cb = None + + self._line_state = 0 # DTR & RTS + # Set a default line coding of 115200/8N1 + self._line_coding = bytearray(b"\x00\xc2\x01\x00\x00\x00\x08") + + self._wb = () # Optional write Buffer (IN endpoint), set by CDC.init() + self._rb = () # Optional read Buffer (OUT endpoint), set by CDC.init() + self._timeout = 1000 # set from CDC.init() as well + + # one control interface endpoint, two data interface endpoints + self.ep_c_in = self.ep_d_in = self.ep_d_out = None + + self._c_itf = None # Number of control interface, data interface is one more + + self.init(**kwargs) + + def init( + self, baudrate=9600, bits=8, parity="N", stop=1, timeout=None, txbuf=256, rxbuf=256, flow=0 + ): + # Configure the CDC serial port. Note that many of these settings like + # baudrate, bits, parity, stop don't change the USB-CDC device behavior + # at all, only the "line coding" as communicated from/to the USB host. + + # Store initial line coding parameters in the USB CDC binary format + # (there is nothing implemented to further change these from Python + # code, the USB host sets them.) + struct.pack_into( + "= _BULK_EP_LEN): + raise ValueError # Buffer sizes are required, rxbuf must be at least one EP + + self._timeout = timeout + self._wb = Buffer(txbuf) + self._rb = Buffer(rxbuf) + + ### + ### Line State & Line Coding State property getters + ### + + @property + def rts(self): + return bool(self._line_state & _LINE_STATE_RTS) + + @property + def dtr(self): + return bool(self._line_state & _LINE_STATE_DTR) + + # Line Coding Representation + # Byte 0-3 Byte 4 Byte 5 Byte 6 + # dwDTERate bCharFormat bParityType bDataBits + + @property + def baudrate(self): + return struct.unpack("= _BULK_EP_LEN + ): + # Can only submit up to the endpoint length per transaction, otherwise we won't + # get any transfer callback until the full transaction completes. + self.submit_xfer(self.ep_d_out, self._rb.pend_write(_BULK_EP_LEN), self._rd_cb) + + def _rd_cb(self, ep, res, num_bytes): + # Whenever a data OUT transfer ends + if res == 0: + self._rb.finish_write(num_bytes) + self._rd_xfer() + + ### + ### io.IOBase stream implementation + ### + + def write(self, buf): + # use a memoryview to track how much of 'buf' we've written so far + # (unfortunately, this means a 1 block allocation for each write, but it's otherwise allocation free.) + start = time.ticks_ms() + mv = memoryview(buf) + while True: + # Keep pushing buf into _wb into it's all gone + nbytes = self._wb.write(mv) + self._wr_xfer() # make sure a transfer is running from _wb + + if nbytes == len(mv): + return len(buf) # Success + + mv = mv[nbytes:] + + # check for timeout + if time.ticks_diff(time.ticks_ms(), start) >= self._timeout: + return len(buf) - len(mv) + + machine.idle() + + def read(self, size): + start = time.ticks_ms() + + # Allocate a suitable buffer to read into + if size >= 0: + b = bytearray(size) + else: + # for size == -1, return however many bytes are ready + b = bytearray(self._rb.readable()) + + n = self._readinto(b, start) + if not n: + return None + if n < len(b): + return b[:n] + return b + + def readinto(self, b): + return self._readinto(b, time.ticks_ms()) + + def _readinto(self, b, start): + if len(b) == 0: + return 0 + + n = 0 + m = memoryview(b) + while n < len(b): + # copy out of the read buffer if there is anything available + if self._rb.readable(): + n += self._rb.readinto(m if n == 0 else m[n:]) + self._rd_xfer() # if _rd was previously full, no transfer will be running + if n == len(b): + break # Done, exit before we call machine.idle() + + if time.ticks_diff(time.ticks_ms(), start) >= self._timeout: + break # Timed out + + machine.idle() + + return n or None + + def ioctl(self, req, arg): + if req == _MP_STREAM_POLL: + return ( + (_MP_STREAM_POLL_WR if (arg & _MP_STREAM_POLL_WR) and self._wb.writable() else 0) + | (_MP_STREAM_POLL_RD if (arg & _MP_STREAM_POLL_RD) and self._rb.readable() else 0) + | + # using the USB level "open" (i.e. connected to host) for !HUP, not !DTR (port is open) + (_MP_STREAM_POLL_HUP if (arg & _MP_STREAM_POLL_HUP) and not self.is_open() else 0) + ) + elif req == _MP_STREAM_FLUSH: + start = time.ticks_ms() + # Wait until write buffer contains no bytes for the lower TinyUSB layer to "read" + while self._wb.readable(): + if not self.is_open(): + return _MP_EINVAL + if time.ticks_diff(time.ticks_ms(), start) > self._timeout: + return _MP_ETIMEDOUT + machine.idle() + return 0 + + return _MP_EINVAL + + def flush(self): + # a C implementation of this exists in stream.c, but it's not in io.IOBase + # and can't immediately be called from here (AFAIK) + r = self.ioctl(_MP_STREAM_FLUSH, 0) + if r: + raise OSError(r) diff --git a/micropython/usb/usb-device-hid/manifest.py b/micropython/usb/usb-device-hid/manifest.py new file mode 100644 index 000000000..af9b8cb84 --- /dev/null +++ b/micropython/usb/usb-device-hid/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0") +require("usb-device") +package("usb") diff --git a/micropython/usb/usb-device-hid/usb/device/hid.py b/micropython/usb/usb-device-hid/usb/device/hid.py new file mode 100644 index 000000000..9e4c70dde --- /dev/null +++ b/micropython/usb/usb-device-hid/usb/device/hid.py @@ -0,0 +1,232 @@ +# MicroPython USB hid module +# +# This implements a base HIDInterface class that can be used directly, +# or subclassed into more specific HID interface types. +# +# MIT license; Copyright (c) 2023 Angus Gratton +from micropython import const +import machine +import struct +import time +from .core import Interface, Descriptor, split_bmRequestType + +_EP_IN_FLAG = const(1 << 7) + +# Control transfer stages +_STAGE_IDLE = const(0) +_STAGE_SETUP = const(1) +_STAGE_DATA = const(2) +_STAGE_ACK = const(3) + +# Request types +_REQ_TYPE_STANDARD = const(0x0) +_REQ_TYPE_CLASS = const(0x1) +_REQ_TYPE_VENDOR = const(0x2) +_REQ_TYPE_RESERVED = const(0x3) + +# Descriptor types +_DESC_HID_TYPE = const(0x21) +_DESC_REPORT_TYPE = const(0x22) +_DESC_PHYSICAL_TYPE = const(0x23) + +# Interface and protocol identifiers +_INTERFACE_CLASS = const(0x03) +_INTERFACE_SUBCLASS_NONE = const(0x00) +_INTERFACE_SUBCLASS_BOOT = const(0x01) + +_INTERFACE_PROTOCOL_NONE = const(0x00) +_INTERFACE_PROTOCOL_KEYBOARD = const(0x01) +_INTERFACE_PROTOCOL_MOUSE = const(0x02) + +# bRequest values for HID control requests +_REQ_CONTROL_GET_REPORT = const(0x01) +_REQ_CONTROL_GET_IDLE = const(0x02) +_REQ_CONTROL_GET_PROTOCOL = const(0x03) +_REQ_CONTROL_GET_DESCRIPTOR = const(0x06) +_REQ_CONTROL_SET_REPORT = const(0x09) +_REQ_CONTROL_SET_IDLE = const(0x0A) +_REQ_CONTROL_SET_PROTOCOL = const(0x0B) + +# Standard descriptor lengths +_STD_DESC_INTERFACE_LEN = const(9) +_STD_DESC_ENDPOINT_LEN = const(7) + + +class HIDInterface(Interface): + # Abstract base class to implement a USB device HID interface in Python. + + def __init__( + self, + report_descriptor, + extra_descriptors=[], + set_report_buf=None, + protocol=_INTERFACE_PROTOCOL_NONE, + interface_str=None, + ): + # Construct a new HID interface. + # + # - report_descriptor is the only mandatory argument, which is the binary + # data consisting of the HID Report Descriptor. See Device Class + # Definition for Human Interface Devices (HID) v1.11 section 6.2.2 Report + # Descriptor, p23. + # + # - extra_descriptors is an optional argument holding additional HID + # descriptors, to append after the mandatory report descriptor. Most + # HID devices do not use these. + # + # - set_report_buf is an optional writable buffer object (i.e. + # bytearray), where SET_REPORT requests from the host can be + # written. Only necessary if the report_descriptor contains Output + # entries. If set, the size must be at least the size of the largest + # Output entry. + # + # - protocol can be set to a specific value as per HID v1.11 section 4.3 Protocols, p9. + # + # - interface_str is an optional string descriptor to associate with the HID USB interface. + super().__init__() + self.report_descriptor = report_descriptor + self.extra_descriptors = extra_descriptors + self._set_report_buf = set_report_buf + self.protocol = protocol + self.interface_str = interface_str + + self._int_ep = None # set during enumeration + + def get_report(self): + return False + + def on_set_report(self, report_data, report_id, report_type): + # Override this function in order to handle SET REPORT requests from the host, + # where it sends data to the HID device. + # + # This function will only be called if the Report descriptor contains at least one Output entry, + # and the set_report_buf argument is provided to the constructor. + # + # Return True to complete the control transfer normally, False to abort it. + return True + + def busy(self): + # Returns True if the interrupt endpoint is busy (i.e. existing transfer is pending) + return self.is_open() and self.xfer_pending(self._int_ep) + + def send_report(self, report_data, timeout_ms=100): + # Helper function to send a HID report in the typical USB interrupt + # endpoint associated with a HID interface. + # + # Returns True if successful, False if HID device is not active or timeout + # is reached without being able to queue the report for sending. + deadline = time.ticks_add(time.ticks_ms(), timeout_ms) + while self.busy(): + if time.ticks_diff(deadline, time.ticks_ms()) <= 0: + return False + machine.idle() + if not self.is_open(): + return False + self.submit_xfer(self._int_ep, report_data) + + def desc_cfg(self, desc, itf_num, ep_num, strs): + # Add the standard interface descriptor + desc.interface( + itf_num, + 1, + _INTERFACE_CLASS, + _INTERFACE_SUBCLASS_NONE, + self.protocol, + len(strs) if self.interface_str else 0, + ) + + if self.interface_str: + strs.append(self.interface_str) + + # As per HID v1.11 section 7.1 Standard Requests, return the contents of + # the standard HID descriptor before the associated endpoint descriptor. + self.get_hid_descriptor(desc) + + # Add the typical single USB interrupt endpoint descriptor associated + # with a HID interface. + self._int_ep = ep_num | _EP_IN_FLAG + desc.endpoint(self._int_ep, "interrupt", 8, 8) + + self.idle_rate = 0 + self.protocol = 0 + + def num_eps(self): + return 1 + + def get_hid_descriptor(self, desc=None): + # Append a full USB HID descriptor from the object's report descriptor + # and optional additional descriptors. + # + # See HID Specification Version 1.1, Section 6.2.1 HID Descriptor p22 + + l = 9 + 3 * len(self.extra_descriptors) # total length + + if desc is None: + desc = Descriptor(bytearray(l)) + + desc.pack( + "> 8 + if desc_type == _DESC_HID_TYPE: + return self.get_hid_descriptor() + if desc_type == _DESC_REPORT_TYPE: + return self.report_descriptor + elif req_type == _REQ_TYPE_CLASS: + # HID Spec p50: 7.2 Class-Specific Requests + if bRequest == _REQ_CONTROL_GET_REPORT: + print("GET_REPORT?") + return False # Unsupported for now + if bRequest == _REQ_CONTROL_GET_IDLE: + return bytes([self.idle_rate]) + if bRequest == _REQ_CONTROL_GET_PROTOCOL: + return bytes([self.protocol]) + if bRequest in (_REQ_CONTROL_SET_IDLE, _REQ_CONTROL_SET_PROTOCOL): + return True + if bRequest == _REQ_CONTROL_SET_REPORT: + return self._set_report_buf # If None, request will stall + return False # Unsupported request + + if stage == _STAGE_ACK: + if req_type == _REQ_TYPE_CLASS: + if bRequest == _REQ_CONTROL_SET_IDLE: + self.idle_rate = wValue >> 8 + elif bRequest == _REQ_CONTROL_SET_PROTOCOL: + self.protocol = wValue + elif bRequest == _REQ_CONTROL_SET_REPORT: + report_id = wValue & 0xFF + report_type = wValue >> 8 + report_data = self._set_report_buf + if wLength < len(report_data): + # need to truncate the response in the callback if we got less bytes + # than allowed for in the buffer + report_data = memoryview(self._set_report_buf)[:wLength] + self.on_set_report(report_data, report_id, report_type) + + return True # allow DATA/ACK stages to complete normally diff --git a/micropython/usb/usb-device-keyboard/manifest.py b/micropython/usb/usb-device-keyboard/manifest.py new file mode 100644 index 000000000..5a2ff307d --- /dev/null +++ b/micropython/usb/usb-device-keyboard/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.1") +require("usb-device-hid") +package("usb") diff --git a/micropython/usb/usb-device-keyboard/usb/device/keyboard.py b/micropython/usb/usb-device-keyboard/usb/device/keyboard.py new file mode 100644 index 000000000..22091c50b --- /dev/null +++ b/micropython/usb/usb-device-keyboard/usb/device/keyboard.py @@ -0,0 +1,233 @@ +# MIT license; Copyright (c) 2023-2024 Angus Gratton +from micropython import const +import time +import usb.device +from usb.device.hid import HIDInterface + +_INTERFACE_PROTOCOL_KEYBOARD = const(0x01) + +_KEY_ARRAY_LEN = const(6) # Size of HID key array, must match report descriptor +_KEY_REPORT_LEN = const(_KEY_ARRAY_LEN + 2) # Modifier Byte + Reserved Byte + Array entries + + +class KeyboardInterface(HIDInterface): + # Synchronous USB keyboard HID interface + + def __init__(self): + super().__init__( + _KEYBOARD_REPORT_DESC, + set_report_buf=bytearray(1), + protocol=_INTERFACE_PROTOCOL_KEYBOARD, + interface_str="MicroPython Keyboard", + ) + self._key_reports = [ + bytearray(_KEY_REPORT_LEN), + bytearray(_KEY_REPORT_LEN), + ] # Ping/pong report buffers + self.numlock = False + + def on_set_report(self, report_data, _report_id, _report_type): + self.on_led_update(report_data[0]) + + def on_led_update(self, led_mask): + # Override to handle keyboard LED updates. led_mask is bitwise ORed + # together values as defined in LEDCode. + pass + + def send_keys(self, down_keys, timeout_ms=100): + # Update the state of the keyboard by sending a report with down_keys + # set, where down_keys is an iterable (list or similar) of integer + # values such as the values defined in KeyCode. + # + # Will block for up to timeout_ms if a previous report is still + # pending to be sent to the host. Returns True on success. + + r, s = self._key_reports # next report buffer to send, spare report buffer + r[0] = 0 # modifier byte + i = 2 # index for next key array item to write to + for k in down_keys: + if k < 0: # Modifier key + r[0] |= -k + elif i < _KEY_REPORT_LEN: + r[i] = k + i += 1 + else: # Excess rollover! Can't report + r[0] = 0 + for i in range(2, _KEY_REPORT_LEN): + r[i] = 0xFF + break + + while i < _KEY_REPORT_LEN: + r[i] = 0 + i += 1 + + if self.send_report(r, timeout_ms): + # Swap buffers if the previous one is newly queued to send, so + # any subsequent call can't modify that buffer mid-send + self._key_reports[0] = s + self._key_reports[1] = r + return True + return False + + +# HID keyboard report descriptor +# +# From p69 of http://www.usb.org/developers/devclass_docs/HID1_11.pdf +# +# fmt: off +_KEYBOARD_REPORT_DESC = ( + b'\x05\x01' # Usage Page (Generic Desktop), + b'\x09\x06' # Usage (Keyboard), + b'\xA1\x01' # Collection (Application), + b'\x05\x07' # Usage Page (Key Codes); + b'\x19\xE0' # Usage Minimum (224), + b'\x29\xE7' # Usage Maximum (231), + b'\x15\x00' # Logical Minimum (0), + b'\x25\x01' # Logical Maximum (1), + b'\x75\x01' # Report Size (1), + b'\x95\x08' # Report Count (8), + b'\x81\x02' # Input (Data, Variable, Absolute), ;Modifier byte + b'\x95\x01' # Report Count (1), + b'\x75\x08' # Report Size (8), + b'\x81\x01' # Input (Constant), ;Reserved byte + b'\x95\x05' # Report Count (5), + b'\x75\x01' # Report Size (1), + b'\x05\x08' # Usage Page (Page# for LEDs), + b'\x19\x01' # Usage Minimum (1), + b'\x29\x05' # Usage Maximum (5), + b'\x91\x02' # Output (Data, Variable, Absolute), ;LED report + b'\x95\x01' # Report Count (1), + b'\x75\x03' # Report Size (3), + b'\x91\x01' # Output (Constant), ;LED report padding + b'\x95\x06' # Report Count (6), + b'\x75\x08' # Report Size (8), + b'\x15\x00' # Logical Minimum (0), + b'\x25\x65' # Logical Maximum(101), + b'\x05\x07' # Usage Page (Key Codes), + b'\x19\x00' # Usage Minimum (0), + b'\x29\x65' # Usage Maximum (101), + b'\x81\x00' # Input (Data, Array), ;Key arrays (6 bytes) + b'\xC0' # End Collection +) +# fmt: on + + +# Standard HID keycodes, as a pseudo-enum class for easy access +# +# Modifier keys are encoded as negative values +class KeyCode: + A = 4 + B = 5 + C = 6 + D = 7 + E = 8 + F = 9 + G = 10 + H = 11 + I = 12 + J = 13 + K = 14 + L = 15 + M = 16 + N = 17 + O = 18 + P = 19 + Q = 20 + R = 21 + S = 22 + T = 23 + U = 24 + V = 25 + W = 26 + X = 27 + Y = 28 + Z = 29 + N1 = 30 # Standard number row keys + N2 = 31 + N3 = 32 + N4 = 33 + N5 = 34 + N6 = 35 + N7 = 36 + N8 = 37 + N9 = 38 + N0 = 39 + ENTER = 40 + ESCAPE = 41 + BACKSPACE = 42 + TAB = 43 + SPACE = 44 + MINUS = 45 # - _ + EQUAL = 46 # = + + OPEN_BRACKET = 47 # [ { + CLOSE_BRACKET = 48 # ] } + BACKSLASH = 49 # \ | + HASH = 50 # # ~ + SEMICOLON = 51 # ; : + QUOTE = 52 # ' " + GRAVE = 53 # ` ~ + COMMA = 54 # , < + DOT = 55 # . > + SLASH = 56 # / ? + CAPS_LOCK = 57 + F1 = 58 + F2 = 59 + F3 = 60 + F4 = 61 + F5 = 62 + F6 = 63 + F7 = 64 + F8 = 65 + F9 = 66 + F10 = 67 + F11 = 68 + F12 = 69 + PRINTSCREEN = 70 + SCROLL_LOCK = 71 + PAUSE = 72 + INSERT = 73 + HOME = 74 + PAGEUP = 75 + DELETE = 76 + END = 77 + PAGEDOWN = 78 + RIGHT = 79 # Arrow keys + LEFT = 80 + DOWN = 81 + UP = 82 + KP_NUM_LOCK = 83 + KP_DIVIDE = 84 + KP_AT = 85 + KP_MULTIPLY = 85 + KP_MINUS = 86 + KP_PLUS = 87 + KP_ENTER = 88 + KP_1 = 89 + KP_2 = 90 + KP_3 = 91 + KP_4 = 92 + KP_5 = 93 + KP_6 = 94 + KP_7 = 95 + KP_8 = 96 + KP_9 = 97 + KP_0 = 98 + + # HID modifier values (negated to allow them to be passed along with the normal keys) + LEFT_CTRL = -0x01 + LEFT_SHIFT = -0x02 + LEFT_ALT = -0x04 + LEFT_UI = -0x08 + RIGHT_CTRL = -0x10 + RIGHT_SHIFT = -0x20 + RIGHT_ALT = -0x40 + RIGHT_UI = -0x80 + + +# HID LED values +class LEDCode: + NUM_LOCK = 0x01 + CAPS_LOCK = 0x02 + SCROLL_LOCK = 0x04 + COMPOSE = 0x08 + KANA = 0x10 diff --git a/micropython/usb/usb-device-midi/manifest.py b/micropython/usb/usb-device-midi/manifest.py new file mode 100644 index 000000000..af9b8cb84 --- /dev/null +++ b/micropython/usb/usb-device-midi/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0") +require("usb-device") +package("usb") diff --git a/micropython/usb/usb-device-midi/usb/device/midi.py b/micropython/usb/usb-device-midi/usb/device/midi.py new file mode 100644 index 000000000..ecb178ea4 --- /dev/null +++ b/micropython/usb/usb-device-midi/usb/device/midi.py @@ -0,0 +1,306 @@ +# MicroPython USB MIDI module +# MIT license; Copyright (c) 2023 Paul Hamshere, 2023-2024 Angus Gratton +from micropython import const, schedule +import struct + +from .core import Interface, Buffer + +_EP_IN_FLAG = const(1 << 7) + +_INTERFACE_CLASS_AUDIO = const(0x01) +_INTERFACE_SUBCLASS_AUDIO_CONTROL = const(0x01) +_INTERFACE_SUBCLASS_AUDIO_MIDISTREAMING = const(0x03) + +# Audio subclass extends the standard endpoint descriptor +# with two extra bytes +_STD_DESC_AUDIO_ENDPOINT_LEN = const(9) +_CLASS_DESC_ENDPOINT_LEN = const(5) + +_STD_DESC_ENDPOINT_TYPE = const(0x5) + +_JACK_TYPE_EMBEDDED = const(0x01) +_JACK_TYPE_EXTERNAL = const(0x02) + +_JACK_IN_DESC_LEN = const(6) +_JACK_OUT_DESC_LEN = const(9) + +# MIDI Status bytes. For Channel messages these are only the upper 4 bits, ORed with the channel number. +# As per https://www.midi.org/specifications-old/item/table-1-summary-of-midi-message +_MIDI_NOTE_OFF = const(0x80) +_MIDI_NOTE_ON = const(0x90) +_MIDI_POLY_KEYPRESS = const(0xA0) +_MIDI_CONTROL_CHANGE = const(0xB0) + +# USB-MIDI CINs (Code Index Numbers), as per USB MIDI Table 4-1 +_CIN_SYS_COMMON_2BYTE = const(0x2) +_CIN_SYS_COMMON_3BYTE = const(0x3) +_CIN_SYSEX_START = const(0x4) +_CIN_SYSEX_END_1BYTE = const(0x5) +_CIN_SYSEX_END_2BYTE = const(0x6) +_CIN_SYSEX_END_3BYTE = const(0x7) +_CIN_NOTE_OFF = const(0x8) +_CIN_NOTE_ON = const(0x9) +_CIN_POLY_KEYPRESS = const(0xA) +_CIN_CONTROL_CHANGE = const(0xB) +_CIN_PROGRAM_CHANGE = const(0xC) +_CIN_CHANNEL_PRESSURE = const(0xD) +_CIN_PITCH_BEND = const(0xE) +_CIN_SINGLE_BYTE = const(0xF) # Not currently supported + +# Jack IDs for a simple bidrectional MIDI device(!) +_EMB_IN_JACK_ID = const(1) +_EXT_IN_JACK_ID = const(2) +_EMB_OUT_JACK_ID = const(3) +_EXT_OUT_JACK_ID = const(4) + +# Data flows, as modelled by USB-MIDI and this hypothetical interface, are as follows: +# Device RX = USB OUT EP => _EMB_IN_JACK => _EMB_OUT_JACK +# Device TX = _EXT_IN_JACK => _EMB_OUT_JACK => USB IN EP + + +class MIDIInterface(Interface): + # Base class to implement a USB MIDI device in Python. + # + # To be compliant this also regisers a dummy USB Audio interface, but that + # interface isn't otherwise used. + + def __init__(self, rxlen=16, txlen=16): + # Arguments are size of transmit and receive buffers in bytes. + + super().__init__() + self.ep_out = None # Set during enumeration. RX direction (host to device) + self.ep_in = None # TX direction (device to host) + self._rx = Buffer(rxlen) + self._tx = Buffer(txlen) + + # Callbacks for handling received MIDI messages. + # + # Subclasses can choose between overriding on_midi_event + # and handling all MIDI events manually, or overriding the + # functions for note on/off and control change, only. + + def on_midi_event(self, cin, midi0, midi1, midi2): + ch = midi0 & 0x0F + if cin == _CIN_NOTE_ON: + self.on_note_on(ch, midi1, midi2) + elif cin == _CIN_NOTE_OFF: + self.on_note_off(ch, midi1, midi2) + elif cin == _CIN_CONTROL_CHANGE: + self.on_control_change(ch, midi1, midi2) + + def on_note_on(self, channel, pitch, vel): + pass # Override to handle Note On messages + + def on_note_off(self, channel, pitch, vel): + pass # Override to handle Note On messages + + def on_control_change(self, channel, controller, value): + pass # Override to handle Control Change messages + + # Helper functions for sending common MIDI messages + + def note_on(self, channel, pitch, vel=0x40): + self.send_event(_CIN_NOTE_ON, _MIDI_NOTE_ON | channel, pitch, vel) + + def note_off(self, channel, pitch, vel=0x40): + self.send_event(_CIN_NOTE_OFF, _MIDI_NOTE_OFF | channel, pitch, vel) + + def control_change(self, channel, controller, value): + self.send_event(_CIN_CONTROL_CHANGE, _MIDI_CONTROL_CHANGE | channel, controller, value) + + def send_event(self, cin, midi0, midi1=0, midi2=0): + # Queue a MIDI Event Packet to send to the host. + # + # CIN = USB-MIDI Code Index Number, see USB MIDI 1.0 section 4 "USB-MIDI Event Packets" + # + # Remaining arguments are 0-3 MIDI data bytes. + # + # Note this function returns when the MIDI Event Packet has been queued, + # not when it's been received by the host. + # + # Returns False if the TX buffer is full and the MIDI Event could not be queued. + w = self._tx.pend_write() + if len(w) < 4: + return False # TX buffer is full. TODO: block here? + w[0] = cin # leave cable number as 0? + w[1] = midi0 + w[2] = midi1 + w[3] = midi2 + self._tx.finish_write(4) + self._tx_xfer() + return True + + def _tx_xfer(self): + # Keep an active IN transfer to send data to the host, whenever + # there is data to send. + if self.is_open() and not self.xfer_pending(self.ep_in) and self._tx.readable(): + self.submit_xfer(self.ep_in, self._tx.pend_read(), self._tx_cb) + + def _tx_cb(self, ep, res, num_bytes): + if res == 0: + self._tx.finish_read(num_bytes) + self._tx_xfer() + + def _rx_xfer(self): + # Keep an active OUT transfer to receive MIDI events from the host + if self.is_open() and not self.xfer_pending(self.ep_out) and self._rx.writable(): + self.submit_xfer(self.ep_out, self._rx.pend_write(), self._rx_cb) + + def _rx_cb(self, ep, res, num_bytes): + if res == 0: + self._rx.finish_write(num_bytes) + schedule(self._on_rx, None) + self._rx_xfer() + + def on_open(self): + super().on_open() + # kick off any transfers that may have queued while the device was not open + self._tx_xfer() + self._rx_xfer() + + def _on_rx(self, _): + # Receive MIDI events. Called via micropython.schedule, outside of the USB callback function. + m = self._rx.pend_read() + i = 0 + while i <= len(m) - 4: + cin = m[i] & 0x0F + self.on_midi_event(cin, m[i + 1], m[i + 2], m[i + 3]) + i += 4 + self._rx.finish_read(i) + + def desc_cfg(self, desc, itf_num, ep_num, strs): + # Start by registering a USB Audio Control interface, that is required to point to the + # actual MIDI interface + desc.interface(itf_num, 0, _INTERFACE_CLASS_AUDIO, _INTERFACE_SUBCLASS_AUDIO_CONTROL) + + # Append the class-specific AudioControl interface descriptor + desc.pack( + "1 USB interface.) + + def __init__(self): + self._open = False + + def desc_cfg(self, desc, itf_num, ep_num, strs): + # Function to build configuration descriptor contents for this interface + # or group of interfaces. This is called on each interface from + # USBDevice.init(). + # + # This function should insert: + # + # - At least one standard Interface descriptor (can call + # - desc.interface()). + # + # Plus, optionally: + # + # - One or more endpoint descriptors (can call desc.endpoint()). + # - An Interface Association Descriptor, prepended before. + # - Other class-specific configuration descriptor data. + # + # This function is called twice per call to USBDevice.init(). The first + # time the values of all arguments are dummies that are used only to + # calculate the total length of the descriptor. Therefore, anything this + # function does should be idempotent and it should add the same + # descriptors each time. If saving interface numbers or endpoint numbers + # for later + # + # Parameters: + # + # - desc - Descriptor helper to write the configuration descriptor bytes into. + # The first time this function is called 'desc' is a dummy object + # with no backing buffer (exists to count the number of bytes needed). + # + # - itf_num - First bNumInterfaces value to assign. The descriptor + # should contain the same number of interfaces returned by num_itfs(), + # starting from this value. + # + # - ep_num - Address of the first available endpoint number to use for + # endpoint descriptor addresses. Subclasses should save the + # endpoint addresses selected, to look up later (although note the first + # time this function is called, the values will be dummies.) + # + # - strs - list of string descriptors for this USB device. This function + # can append to this list, and then insert the index of the new string + # in the list into the configuration descriptor. + raise NotImplementedError + + def num_itfs(self): + # Return the number of actual USB Interfaces represented by this object + # (as set in desc_cfg().) + # + # Only needs to be overriden if implementing a Interface class that + # represents more than one USB Interface descriptor (i.e. MIDI), or an + # Interface Association Descriptor (i.e. USB-CDC). + return 1 + + def num_eps(self): + # Return the number of USB Endpoint numbers represented by this object + # (as set in desc_cfg().) + # + # Note for each count returned by this function, the interface may + # choose to have both an IN and OUT endpoint (i.e. IN flag is not + # considered a value here.) + # + # This value can be zero, if the USB Host only communicates with this + # interface using control transfers. + return 0 + + def on_open(self): + # Callback called when the USB host accepts the device configuration. + # + # Override this function to initiate any operations that the USB interface + # should do when the USB device is configured to the host. + self._open = True + + def on_reset(self): + # Callback called on every registered interface when the USB device is + # reset by the host. This can happen when the USB device is unplugged, + # or if the host triggers a reset for some other reason. + # + # Override this function to cancel any pending operations specific to + # the interface (outstanding USB transfers are already cancelled). + # + # At this point, no USB functionality is available - on_open() will + # be called later if/when the USB host re-enumerates and configures the + # interface. + self._open = False + + def is_open(self): + # Returns True if the interface has been configured by the host and is in + # active use. + return self._open + + def on_device_control_xfer(self, stage, request): + # Control transfer callback. Override to handle a non-standard device + # control transfer where bmRequestType Recipient is Device, Type is + # utils.REQ_TYPE_CLASS, and the lower byte of wIndex indicates this interface. + # + # (See USB 2.0 specification 9.4 Standard Device Requests, p250). + # + # This particular request type seems pretty uncommon for a device class + # driver to need to handle, most hosts will not send this so most + # implementations won't need to override it. + # + # Parameters: + # + # - stage is one of utils.STAGE_SETUP, utils.STAGE_DATA, utils.STAGE_ACK. + # + # - request is a memoryview into a USB request packet, as per USB 2.0 + # specification 9.3 USB Device Requests, p250. the memoryview is only + # valid while the callback is running. + # + # The function can call split_bmRequestType(request[0]) to split + # bmRequestType into (Recipient, Type, Direction). + # + # Result, any of: + # + # - True to continue the request, False to STALL the endpoint. + # - Buffer interface object to provide a buffer to the host as part of the + # transfer, if applicable. + return False + + def on_interface_control_xfer(self, stage, request): + # Control transfer callback. Override to handle a device control + # transfer where bmRequestType Recipient is Interface, and the lower byte + # of wIndex indicates this interface. + # + # (See USB 2.0 specification 9.4 Standard Device Requests, p250). + # + # bmRequestType Type field may have different values. It's not necessary + # to handle the mandatory Standard requests (bmRequestType Type == + # utils.REQ_TYPE_STANDARD), if the driver returns False in these cases then + # TinyUSB will provide the necessary responses. + # + # See on_device_control_xfer() for a description of the arguments and + # possible return values. + return False + + def on_endpoint_control_xfer(self, stage, request): + # Control transfer callback. Override to handle a device + # control transfer where bmRequestType Recipient is Endpoint and + # the lower byte of wIndex indicates an endpoint address associated + # with this interface. + # + # bmRequestType Type will generally have any value except + # utils.REQ_TYPE_STANDARD, as Standard endpoint requests are handled by + # TinyUSB. The exception is the the Standard "Set Feature" request. This + # is handled by Tiny USB but also passed through to the driver in case it + # needs to change any internal state, but most drivers can ignore and + # return False in this case. + # + # (See USB 2.0 specification 9.4 Standard Device Requests, p250). + # + # See on_device_control_xfer() for a description of the parameters and + # possible return values. + return False + + def xfer_pending(self, ep_addr): + # Return True if a transfer is already pending on ep_addr. + # + # Only one transfer can be submitted at a time. + # + # The transfer is marked pending while a completion callback is running + # for that endpoint, unless this function is called from the callback + # itself. This makes it simple to submit a new transfer from the + # completion callback. + return _dev and _dev._xfer_pending(ep_addr) + + def submit_xfer(self, ep_addr, data, done_cb=None): + # Submit a USB transfer (of any type except control) + # + # Parameters: + # + # - ep_addr. Address of the endpoint to submit the transfer on. Caller is + # responsible for ensuring that ep_addr is correct and belongs to this + # interface. Only one transfer can be active at a time on each endpoint. + # + # - data. Buffer containing data to send, or for data to be read into + # (depending on endpoint direction). + # + # - done_cb. Optional callback function for when the transfer + # completes. The callback is called with arguments (ep_addr, result, + # xferred_bytes) where result is one of xfer_result_t enum (see top of + # this file), and xferred_bytes is an integer. + # + # If the function returns, the transfer is queued. + # + # The function will raise RuntimeError under the following conditions: + # + # - The interface is not "open" (i.e. has not been enumerated and configured + # by the host yet.) + # + # - A transfer is already pending on this endpoint (use xfer_pending() to check + # before sending if needed.) + # + # - A DCD error occurred when queueing the transfer on the hardware. + # + # + # Will raise TypeError if 'data' isn't he correct type of buffer for the + # endpoint transfer direction. + # + # Note that done_cb may be called immediately, possibly before this + # function has returned to the caller. + if not self._open: + raise RuntimeError("Not open") + _dev._submit_xfer(ep_addr, data, done_cb) + + def stall(self, ep_addr, *args): + # Set or get the endpoint STALL state. + # + # To get endpoint stall stage, call with a single argument. + # To set endpoint stall state, call with an additional boolean + # argument to set or clear. + # + # Generally endpoint STALL is handled automatically, but there are some + # device classes that need to explicitly stall or unstall an endpoint + # under certain conditions. + if not self._open or ep_addr not in self._eps: + raise RuntimeError + _dev._usbd.stall(ep_addr, *args) + + +class Descriptor: + # Wrapper class for writing a descriptor in-place into a provided buffer + # + # Doesn't resize the buffer. + # + # Can be initialised with b=None to perform a dummy pass that calculates the + # length needed for the buffer. + def __init__(self, b): + self.b = b + self.o = 0 # offset of data written to the buffer + + def pack(self, fmt, *args): + # Utility function to pack new data into the descriptor + # buffer, starting at the current offset. + # + # Arguments are the same as struct.pack(), but it fills the + # pre-allocated descriptor buffer (growing if needed), instead of + # returning anything. + self.pack_into(fmt, self.o, *args) + + def pack_into(self, fmt, offs, *args): + # Utility function to pack new data into the descriptor at offset 'offs'. + # + # If the data written is before 'offs' then self.o isn't incremented, + # otherwise it's incremented to point at the end of the written data. + end = offs + struct.calcsize(fmt) + if self.b: + struct.pack_into(fmt, self.b, offs, *args) + self.o = max(self.o, end) + + def extend(self, a): + # Extend the descriptor with some bytes-like data + if self.b: + self.b[self.o : self.o + len(a)] = a + self.o += len(a) + + # TODO: At the moment many of these arguments are named the same as the relevant field + # in the spec, as this is easier to understand. Can save some code size by collapsing them + # down. + + def interface( + self, + bInterfaceNumber, + bNumEndpoints, + bInterfaceClass=_INTERFACE_CLASS_VENDOR, + bInterfaceSubClass=_INTERFACE_SUBCLASS_NONE, + bInterfaceProtocol=_PROTOCOL_NONE, + iInterface=0, + ): + # Utility function to append a standard Interface descriptor, with + # the properties specified in the parameter list. + # + # Defaults for bInterfaceClass, SubClass and Protocol are a "vendor" + # device. + # + # Note that iInterface is a string index number. If set, it should be set + # by the caller Interface to the result of self._get_str_index(s), + # where 's' is a string found in self.strs. + self.pack( + "BBBBBBBBB", + _STD_DESC_INTERFACE_LEN, # bLength + _STD_DESC_INTERFACE_TYPE, # bDescriptorType + bInterfaceNumber, + 0, # bAlternateSetting, not currently supported + bNumEndpoints, + bInterfaceClass, + bInterfaceSubClass, + bInterfaceProtocol, + iInterface, + ) + + def endpoint(self, bEndpointAddress, bmAttributes, wMaxPacketSize, bInterval=1): + # Utility function to append a standard Endpoint descriptor, with + # the properties specified in the parameter list. + # + # See USB 2.0 specification section 9.6.6 Endpoint p269 + # + # As well as a numeric value, bmAttributes can be a string value to represent + # common endpoint types: "control", "bulk", "interrupt". + if bmAttributes == "control": + bmAttributes = 0 + elif bmAttributes == "bulk": + bmAttributes = 2 + elif bmAttributes == "interrupt": + bmAttributes = 3 + + self.pack( + "> 5) & 0x03, + (bmRequestType >> 7) & 0x01, + ) + + +class Buffer: + # An interrupt-safe producer/consumer buffer that wraps a bytearray object. + # + # Kind of like a ring buffer, but supports the idea of returning a + # memoryview for either read or write of multiple bytes (suitable for + # passing to a buffer function without needing to allocate another buffer to + # read into.) + # + # Consumer can call pend_read() to get a memoryview to read from, and then + # finish_read(n) when done to indicate it read 'n' bytes from the + # memoryview. There is also a readinto() convenience function. + # + # Producer must call pend_write() to get a memorybuffer to write into, and + # then finish_write(n) when done to indicate it wrote 'n' bytes into the + # memoryview. There is also a normal write() convenience function. + # + # - Only one producer and one consumer is supported. + # + # - Calling pend_read() and pend_write() is effectively idempotent, they can be + # called more than once without a corresponding finish_x() call if necessary + # (provided only one thread does this, as per the previous point.) + # + # - Calling finish_write() and finish_read() is hard interrupt safe (does + # not allocate). pend_read() and pend_write() each allocate 1 block for + # the memoryview that is returned. + # + # The buffer contents are always laid out as: + # + # - Slice [:_n] = bytes of valid data waiting to read + # - Slice [_n:_w] = unused space + # - Slice [_w:] = bytes of pending write buffer waiting to be written + # + # This buffer should be fast when most reads and writes are balanced and use + # the whole buffer. When this doesn't happen, performance degrades to + # approximate a Python-based single byte ringbuffer. + # + def __init__(self, length): + self._b = memoryview(bytearray(length)) + # number of bytes in buffer read to read, starting at index 0. Updated + # by both producer & consumer. + self._n = 0 + # start index of a pending write into the buffer, if any. equals + # len(self._b) if no write is pending. Updated by producer only. + self._w = length + + def writable(self): + # Number of writable bytes in the buffer. Assumes no pending write is outstanding. + return len(self._b) - self._n + + def readable(self): + # Number of readable bytes in the buffer. Assumes no pending read is outstanding. + return self._n + + def pend_write(self, wmax=None): + # Returns a memoryview that the producer can write bytes into. + # start the write at self._n, the end of data waiting to read + # + # If wmax is set then the memoryview is pre-sliced to be at most + # this many bytes long. + # + # (No critical section needed as self._w is only updated by the producer.) + self._w = self._n + end = (self._w + wmax) if wmax else len(self._b) + return self._b[self._w : end] + + def finish_write(self, nbytes): + # Called by the producer to indicate it wrote nbytes into the buffer. + ist = machine.disable_irq() + try: + assert nbytes <= len(self._b) - self._w # can't say we wrote more than was pended + if self._n == self._w: + # no data was read while the write was happening, so the buffer is already in place + # (this is the fast path) + self._n += nbytes + else: + # Slow path: data was read while the write was happening, so + # shuffle the newly written bytes back towards index 0 to avoid fragmentation + # + # As this updates self._n we have to do it in the critical + # section, so do it byte by byte to avoid allocating. + while nbytes > 0: + self._b[self._n] = self._b[self._w] + self._n += 1 + self._w += 1 + nbytes -= 1 + + self._w = len(self._b) + finally: + machine.enable_irq(ist) + + def write(self, w): + # Helper method for the producer to write into the buffer in one call + pw = self.pend_write() + to_w = min(len(w), len(pw)) + if to_w: + pw[:to_w] = w[:to_w] + self.finish_write(to_w) + return to_w + + def pend_read(self): + # Return a memoryview slice that the consumer can read bytes from + return self._b[: self._n] + + def finish_read(self, nbytes): + # Called by the consumer to indicate it read nbytes from the buffer. + if not nbytes: + return + ist = machine.disable_irq() + try: + assert nbytes <= self._n # can't say we read more than was available + i = 0 + self._n -= nbytes + while i < self._n: + # consumer only read part of the buffer, so shuffle remaining + # read data back towards index 0 to avoid fragmentation + self._b[i] = self._b[i + nbytes] + i += 1 + finally: + machine.enable_irq(ist) + + def readinto(self, b): + # Helper method for the consumer to read out of the buffer in one call + pr = self.pend_read() + to_r = min(len(pr), len(b)) + if to_r: + b[:to_r] = pr[:to_r] + self.finish_read(to_r) + return to_r diff --git a/pyproject.toml b/pyproject.toml index 3b2524545..4776ddfe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,16 +54,12 @@ select = [ # "UP", # pyupgrade ] ignore = [ - "E401", - "E402", "E722", - "E741", + "E741", # 'l' is currently widely used "F401", "F403", "F405", - "E501", - "F541", - "F841", + "E501", # line length, recommended to disable "ISC001", "ISC003", # micropython does not support implicit concatenation of f-strings "PIE810", # micropython does not support passing tuples to .startswith or .endswith @@ -76,7 +72,7 @@ ignore = [ "PLW2901", "RUF012", "RUF100", - "W191", + "W191", # tab-indent, redundant when using formatter ] line-length = 99 target-version = "py37" diff --git a/python-ecosys/aiohttp/aiohttp/__init__.py b/python-ecosys/aiohttp/aiohttp/__init__.py index d31788435..1565163c4 100644 --- a/python-ecosys/aiohttp/aiohttp/__init__.py +++ b/python-ecosys/aiohttp/aiohttp/__init__.py @@ -22,7 +22,8 @@ def _decode(self, data): c_encoding = self.headers.get("Content-Encoding") if c_encoding in ("gzip", "deflate", "gzip,deflate"): try: - import deflate, io + import deflate + import io if c_encoding == "deflate": with deflate.DeflateIO(io.BytesIO(data), deflate.ZLIB) as d: @@ -38,10 +39,10 @@ async def read(self, sz=-1): return self._decode(await self.content.read(sz)) async def text(self, encoding="utf-8"): - return (await self.read(sz=-1)).decode(encoding) + return (await self.read(int(self.headers.get("Content-Length", -1)))).decode(encoding) async def json(self): - return _json.loads(await self.read()) + return _json.loads(await self.read(int(self.headers.get("Content-Length", -1)))) def __repr__(self): return "" % (self.status, self.headers) @@ -104,7 +105,6 @@ async def __aexit__(self, *args): async def _request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}): redir_cnt = 0 - redir_url = None while redir_cnt < 2: reader = await self.request_raw(method, url, data, json, ssl, params, headers) _headers = [] @@ -121,7 +121,7 @@ async def _request(self, method, url, data=None, json=None, ssl=None, params=Non if b"chunked" in line: chunked = True elif line.startswith(b"Location:"): - url = line.rstrip().split(None, 1)[1].decode("latin-1") + url = line.rstrip().split(None, 1)[1].decode() if 301 <= status <= 303: redir_cnt += 1 @@ -195,17 +195,22 @@ async def request_raw( if "Host" not in headers: headers.update(Host=host) if not data: - query = "%s /%s %s\r\n%s\r\n" % ( + query = b"%s /%s %s\r\n%s\r\n" % ( method, path, version, "\r\n".join(f"{k}: {v}" for k, v in headers.items()) + "\r\n" if headers else "", ) else: - headers.update(**{"Content-Length": len(str(data))}) if json: headers.update(**{"Content-Type": "application/json"}) - query = """%s /%s %s\r\n%s\r\n%s\r\n\r\n""" % ( + if isinstance(data, bytes): + headers.update(**{"Content-Type": "application/octet-stream"}) + else: + data = data.encode() + + headers.update(**{"Content-Length": len(data)}) + query = b"""%s /%s %s\r\n%s\r\n%s""" % ( method, path, version, @@ -213,10 +218,10 @@ async def request_raw( data, ) if not is_handshake: - await writer.awrite(query.encode("latin-1")) + await writer.awrite(query) return reader else: - await writer.awrite(query.encode()) + await writer.awrite(query) return reader, writer def request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}): diff --git a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py index e5575a11c..07d833730 100644 --- a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py +++ b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py @@ -86,7 +86,7 @@ def _parse_frame_header(cls, header): def _process_websocket_frame(self, opcode, payload): if opcode == self.TEXT: - payload = payload.decode() + payload = str(payload, "utf-8") elif opcode == self.BINARY: pass elif opcode == self.CLOSE: @@ -143,7 +143,7 @@ async def handshake(self, uri, ssl, req): headers["Host"] = f"{uri.hostname}:{uri.port}" headers["Connection"] = "Upgrade" headers["Upgrade"] = "websocket" - headers["Sec-WebSocket-Key"] = key + headers["Sec-WebSocket-Key"] = str(key, "utf-8") headers["Sec-WebSocket-Version"] = "13" headers["Origin"] = f"{_http_proto}://{uri.hostname}:{uri.port}" diff --git a/python-ecosys/aiohttp/examples/client.py b/python-ecosys/aiohttp/examples/client.py index 471737b26..0a6476064 100644 --- a/python-ecosys/aiohttp/examples/client.py +++ b/python-ecosys/aiohttp/examples/client.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/compression.py b/python-ecosys/aiohttp/examples/compression.py index 21f9cf7fd..a1c6276b2 100644 --- a/python-ecosys/aiohttp/examples/compression.py +++ b/python-ecosys/aiohttp/examples/compression.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/get.py b/python-ecosys/aiohttp/examples/get.py index 43507a6e7..087d6fb51 100644 --- a/python-ecosys/aiohttp/examples/get.py +++ b/python-ecosys/aiohttp/examples/get.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/headers.py b/python-ecosys/aiohttp/examples/headers.py index c3a92fc49..ec5c00a80 100644 --- a/python-ecosys/aiohttp/examples/headers.py +++ b/python-ecosys/aiohttp/examples/headers.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/methods.py b/python-ecosys/aiohttp/examples/methods.py index 118777c4e..af38ff652 100644 --- a/python-ecosys/aiohttp/examples/methods.py +++ b/python-ecosys/aiohttp/examples/methods.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/params.py b/python-ecosys/aiohttp/examples/params.py index 8c47e2097..9aecb2ab8 100644 --- a/python-ecosys/aiohttp/examples/params.py +++ b/python-ecosys/aiohttp/examples/params.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/ws.py b/python-ecosys/aiohttp/examples/ws.py index e989a39c5..b96ee6819 100644 --- a/python-ecosys/aiohttp/examples/ws.py +++ b/python-ecosys/aiohttp/examples/ws.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/ws_repl_echo.py b/python-ecosys/aiohttp/examples/ws_repl_echo.py index 9393620e3..c41a4ee5e 100644 --- a/python-ecosys/aiohttp/examples/ws_repl_echo.py +++ b/python-ecosys/aiohttp/examples/ws_repl_echo.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/manifest.py b/python-ecosys/aiohttp/manifest.py index d68039c95..748970e5b 100644 --- a/python-ecosys/aiohttp/manifest.py +++ b/python-ecosys/aiohttp/manifest.py @@ -1,6 +1,6 @@ metadata( description="HTTP client module for MicroPython asyncio module", - version="0.0.1", + version="0.0.3", pypi="aiohttp", ) diff --git a/python-ecosys/cbor2/cbor2/__init__.py b/python-ecosys/cbor2/cbor2/__init__.py index 40114d8b2..7cd98734e 100644 --- a/python-ecosys/cbor2/cbor2/__init__.py +++ b/python-ecosys/cbor2/cbor2/__init__.py @@ -24,5 +24,10 @@ """ -from . import decoder -from . import encoder +from ._decoder import CBORDecoder +from ._decoder import load +from ._decoder import loads + +from ._encoder import CBOREncoder +from ._encoder import dump +from ._encoder import dumps diff --git a/python-ecosys/cbor2/cbor2/decoder.py b/python-ecosys/cbor2/cbor2/_decoder.py similarity index 100% rename from python-ecosys/cbor2/cbor2/decoder.py rename to python-ecosys/cbor2/cbor2/_decoder.py diff --git a/python-ecosys/cbor2/cbor2/encoder.py b/python-ecosys/cbor2/cbor2/_encoder.py similarity index 100% rename from python-ecosys/cbor2/cbor2/encoder.py rename to python-ecosys/cbor2/cbor2/_encoder.py diff --git a/python-ecosys/cbor2/examples/cbor_test.py b/python-ecosys/cbor2/examples/cbor_test.py index 79ae6089e..b4f351786 100644 --- a/python-ecosys/cbor2/examples/cbor_test.py +++ b/python-ecosys/cbor2/examples/cbor_test.py @@ -24,16 +24,15 @@ """ -from cbor2 import encoder -from cbor2 import decoder +import cbor2 input = [ {"bn": "urn:dev:ow:10e2073a01080063", "u": "Cel", "t": 1.276020076e09, "v": 23.5}, {"u": "Cel", "t": 1.276020091e09, "v": 23.6}, ] -data = encoder.dumps(input) +data = cbor2.dumps(input) print(data) print(data.hex()) -text = decoder.loads(data) +text = cbor2.loads(data) print(text) diff --git a/python-ecosys/cbor2/manifest.py b/python-ecosys/cbor2/manifest.py index b94ecc886..aa4b77092 100644 --- a/python-ecosys/cbor2/manifest.py +++ b/python-ecosys/cbor2/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.0", pypi="cbor2") +metadata(version="1.0.0", pypi="cbor2") package("cbor2") diff --git a/python-ecosys/iperf3/iperf3.py b/python-ecosys/iperf3/iperf3.py index a5c54445d..363d10d59 100644 --- a/python-ecosys/iperf3/iperf3.py +++ b/python-ecosys/iperf3/iperf3.py @@ -12,9 +12,12 @@ iperf3.client('192.168.1.5', udp=True, reverse=True) """ -import sys, struct -import time, select, socket import json +import select +import socket +import struct +import sys +import time # Provide a urandom() function, supporting devices without os.urandom(). try: diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index 1c46a7384..eb7bb2d42 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.8.1", pypi="requests") +metadata(version="0.10.0", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index fd751e623..a9a183619 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -1,4 +1,4 @@ -import usocket +import socket class Response: @@ -28,9 +28,9 @@ def text(self): return str(self.content, self.encoding) def json(self): - import ujson + import json - return ujson.loads(self.content) + return json.loads(self.content) def request( @@ -38,21 +38,24 @@ def request( url, data=None, json=None, - headers={}, + headers=None, stream=None, auth=None, timeout=None, parse_headers=True, ): + if headers is None: + headers = {} + redirect = None # redirection url, None means no redirection chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None) if auth is not None: - import ubinascii + import binascii username, password = auth formated = b"{}:{}".format(username, password) - formated = str(ubinascii.b2a_base64(formated)[:-1], "ascii") + formated = str(binascii.b2a_base64(formated)[:-1], "ascii") headers["Authorization"] = "Basic {}".format(formated) try: @@ -63,7 +66,7 @@ def request( if proto == "http:": port = 80 elif proto == "https:": - import ussl + import tls port = 443 else: @@ -73,14 +76,14 @@ def request( host, port = host.split(":", 1) port = int(port) - ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] resp_d = None if parse_headers is not False: resp_d = {} - s = usocket.socket(ai[0], usocket.SOCK_STREAM, ai[2]) + s = socket.socket(ai[0], socket.SOCK_STREAM, ai[2]) if timeout is not None: # Note: settimeout is not supported on all platforms, will raise @@ -90,35 +93,53 @@ def request( try: s.connect(ai[-1]) if proto == "https:": - s = ussl.wrap_socket(s, server_hostname=host) + context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + context.verify_mode = tls.CERT_NONE + s = context.wrap_socket(s, server_hostname=host) s.write(b"%s /%s HTTP/1.0\r\n" % (method, path)) + if "Host" not in headers: - s.write(b"Host: %s\r\n" % host) + headers["Host"] = host + + if json is not None: + assert data is None + from json import dumps + + data = dumps(json) + + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + + if data: + if chunked_data: + if "Transfer-Encoding" not in headers and "Content-Length" not in headers: + headers["Transfer-Encoding"] = "chunked" + elif "Content-Length" not in headers: + headers["Content-Length"] = str(len(data)) + + if "Connection" not in headers: + headers["Connection"] = "close" + # Iterate over keys to avoid tuple alloc for k in headers: s.write(k) s.write(b": ") s.write(headers[k]) s.write(b"\r\n") - if json is not None: - assert data is None - import ujson - data = ujson.dumps(json) - s.write(b"Content-Type: application/json\r\n") - if data: - if chunked_data: - s.write(b"Transfer-Encoding: chunked\r\n") - else: - s.write(b"Content-Length: %d\r\n" % len(data)) - s.write(b"Connection: close\r\n\r\n") + s.write(b"\r\n") + if data: if chunked_data: - for chunk in data: - s.write(b"%x\r\n" % len(chunk)) - s.write(chunk) - s.write(b"\r\n") - s.write("0\r\n\r\n") + if headers.get("Transfer-Encoding", None) == "chunked": + for chunk in data: + s.write(b"%x\r\n" % len(chunk)) + s.write(chunk) + s.write(b"\r\n") + s.write("0\r\n\r\n") + else: + for chunk in data: + s.write(chunk) else: s.write(data) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py new file mode 100644 index 000000000..513e533a3 --- /dev/null +++ b/python-ecosys/requests/test_requests.py @@ -0,0 +1,155 @@ +import io +import sys + + +class Socket: + def __init__(self): + self._write_buffer = io.BytesIO() + self._read_buffer = io.BytesIO(b"HTTP/1.0 200 OK\r\n\r\n") + + def connect(self, address): + pass + + def write(self, buf): + self._write_buffer.write(buf) + + def readline(self): + return self._read_buffer.readline() + + +class socket: + AF_INET = 2 + SOCK_STREAM = 1 + IPPROTO_TCP = 6 + + @staticmethod + def getaddrinfo(host, port, af=0, type=0, flags=0): + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 80))] + + def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): + return Socket() + + +sys.modules["socket"] = socket +# ruff: noqa: E402 +import requests + + +def format_message(response): + return response.raw._write_buffer.getvalue().decode("utf8") + + +def test_simple_get(): + response = requests.request("GET", "http://example.com") + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n" + ), format_message(response) + + +def test_get_auth(): + response = requests.request( + "GET", "http://example.com", auth=("test-username", "test-password") + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Host: example.com\r\n" + + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n" + + b"Connection: close\r\n\r\n" + ), format_message(response) + + +def test_get_custom_header(): + response = requests.request("GET", "http://example.com", headers={"User-Agent": "test-agent"}) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"User-Agent: test-agent\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + ), format_message(response) + + +def test_post_json(): + response = requests.request("GET", "http://example.com", json="test") + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Connection: close\r\n" + + b"Content-Type: application/json\r\n" + + b"Host: example.com\r\n" + + b"Content-Length: 6\r\n\r\n" + + b'"test"' + ), format_message(response) + + +def test_post_chunked_data(): + def chunks(): + yield "test" + + response = requests.request("GET", "http://example.com", data=chunks()) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Transfer-Encoding: chunked\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + + b"4\r\ntest\r\n" + + b"0\r\n\r\n" + ), format_message(response) + + +def test_overwrite_get_headers(): + response = requests.request( + "GET", "http://example.com", headers={"Connection": "keep-alive", "Host": "test.com"} + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + b"Host: test.com\r\n" + b"Connection: keep-alive\r\n\r\n" + ), format_message(response) + + +def test_overwrite_post_json_headers(): + response = requests.request( + "GET", + "http://example.com", + json="test", + headers={"Content-Type": "text/plain", "Content-Length": "10"}, + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Connection: close\r\n" + + b"Content-Length: 10\r\n" + + b"Content-Type: text/plain\r\n" + + b"Host: example.com\r\n\r\n" + + b'"test"' + ), format_message(response) + + +def test_overwrite_post_chunked_data_headers(): + def chunks(): + yield "test" + + response = requests.request( + "GET", "http://example.com", data=chunks(), headers={"Content-Length": "4"} + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Host: example.com\r\n" + + b"Content-Length: 4\r\n" + + b"Connection: close\r\n\r\n" + + b"test" + ), format_message(response) + + +test_simple_get() +test_get_auth() +test_get_custom_header() +test_post_json() +test_post_chunked_data() +test_overwrite_get_headers() +test_overwrite_post_json_headers() +test_overwrite_post_chunked_data_headers() diff --git a/python-stdlib/argparse/argparse.py b/python-stdlib/argparse/argparse.py index cb575dd24..5c92887f9 100644 --- a/python-stdlib/argparse/argparse.py +++ b/python-stdlib/argparse/argparse.py @@ -3,7 +3,7 @@ """ import sys -from ucollections import namedtuple +from collections import namedtuple class _ArgError(BaseException): diff --git a/python-stdlib/base64/manifest.py b/python-stdlib/base64/manifest.py index 613d3bc62..9e1b31751 100644 --- a/python-stdlib/base64/manifest.py +++ b/python-stdlib/base64/manifest.py @@ -1,6 +1,5 @@ -metadata(version="3.3.5") +metadata(version="3.3.6") require("binascii") -require("struct") module("base64.py") diff --git a/python-stdlib/binascii/test_binascii.py b/python-stdlib/binascii/test_binascii.py index 942ddc51b..075b2ff3c 100644 --- a/python-stdlib/binascii/test_binascii.py +++ b/python-stdlib/binascii/test_binascii.py @@ -1,5 +1,5 @@ from binascii import * -import utime +import time data = b"zlutoucky kun upel dabelske ody" h = hexlify(data) @@ -14,10 +14,10 @@ a2b_base64(b"as==") == b"j" -start = utime.time() +start = time.time() for x in range(100000): d = unhexlify(h) -print("100000 iterations in: " + str(utime.time() - start)) +print("100000 iterations in: " + str(time.time() - start)) print("OK") diff --git a/python-stdlib/collections-deque/collections/deque.py b/python-stdlib/collections-deque/collections/deque.py deleted file mode 100644 index 1d8c62d4b..000000000 --- a/python-stdlib/collections-deque/collections/deque.py +++ /dev/null @@ -1,36 +0,0 @@ -class deque: - def __init__(self, iterable=None): - if iterable is None: - self.q = [] - else: - self.q = list(iterable) - - def popleft(self): - return self.q.pop(0) - - def popright(self): - return self.q.pop() - - def pop(self): - return self.q.pop() - - def append(self, a): - self.q.append(a) - - def appendleft(self, a): - self.q.insert(0, a) - - def extend(self, a): - self.q.extend(a) - - def __len__(self): - return len(self.q) - - def __bool__(self): - return bool(self.q) - - def __iter__(self): - yield from self.q - - def __str__(self): - return "deque({})".format(self.q) diff --git a/python-stdlib/collections-deque/manifest.py b/python-stdlib/collections-deque/manifest.py deleted file mode 100644 index 0133d2bad..000000000 --- a/python-stdlib/collections-deque/manifest.py +++ /dev/null @@ -1,4 +0,0 @@ -metadata(version="0.1.3") - -require("collections") -package("collections") diff --git a/python-stdlib/collections/collections/__init__.py b/python-stdlib/collections/collections/__init__.py index 7f3be5673..36dfc1c41 100644 --- a/python-stdlib/collections/collections/__init__.py +++ b/python-stdlib/collections/collections/__init__.py @@ -6,10 +6,6 @@ from .defaultdict import defaultdict except ImportError: pass -try: - from .deque import deque -except ImportError: - pass class MutableMapping: diff --git a/python-stdlib/collections/manifest.py b/python-stdlib/collections/manifest.py index d5ef69472..0ce56d1fa 100644 --- a/python-stdlib/collections/manifest.py +++ b/python-stdlib/collections/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.2.0") package("collections") diff --git a/python-stdlib/contextlib/contextlib.py b/python-stdlib/contextlib/contextlib.py index 20c21acaa..1dbb905bf 100644 --- a/python-stdlib/contextlib/contextlib.py +++ b/python-stdlib/contextlib/contextlib.py @@ -85,13 +85,13 @@ class ExitStack(object): """ def __init__(self): - self._exit_callbacks = deque() + self._exit_callbacks = [] def pop_all(self): """Preserve the context stack by transferring it to a new instance""" new_stack = type(self)() new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() + self._exit_callbacks = [] return new_stack def _push_cm_exit(self, cm, cm_exit): diff --git a/python-stdlib/contextlib/manifest.py b/python-stdlib/contextlib/manifest.py index 2894ec5c4..3e05bca18 100644 --- a/python-stdlib/contextlib/manifest.py +++ b/python-stdlib/contextlib/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Port of contextlib for micropython", version="3.4.3") +metadata(description="Port of contextlib for micropython", version="3.4.4") require("ucontextlib") require("collections") diff --git a/python-stdlib/contextlib/tests.py b/python-stdlib/contextlib/tests.py index 19f07add8..c122c452e 100644 --- a/python-stdlib/contextlib/tests.py +++ b/python-stdlib/contextlib/tests.py @@ -399,7 +399,7 @@ def test_exit_exception_chaining_suppress(self): def test_excessive_nesting(self): # The original implementation would die with RecursionError here with ExitStack() as stack: - for i in range(10000): + for i in range(5000): stack.callback(int) def test_instance_bypass(self): diff --git a/python-stdlib/copy/copy.py b/python-stdlib/copy/copy.py index f7bfdd6a1..0a9283777 100644 --- a/python-stdlib/copy/copy.py +++ b/python-stdlib/copy/copy.py @@ -62,7 +62,7 @@ class Error(Exception): error = Error # backward compatibility try: - from ucollections import OrderedDict + from collections import OrderedDict except ImportError: OrderedDict = None diff --git a/python-stdlib/datetime/test_datetime.py b/python-stdlib/datetime/test_datetime.py index 372bdf3de..98da458f9 100644 --- a/python-stdlib/datetime/test_datetime.py +++ b/python-stdlib/datetime/test_datetime.py @@ -2082,9 +2082,11 @@ def test_timetuple00(self): with LocalTz("Europe/Rome"): self.assertEqual(dt1.timetuple()[:8], (2002, 1, 31, 0, 0, 0, 3, 31)) + @unittest.skip("broken when running with non-UTC timezone") def test_timetuple01(self): self.assertEqual(dt27tz2.timetuple()[:8], (2010, 3, 27, 12, 0, 0, 5, 86)) + @unittest.skip("broken when running with non-UTC timezone") def test_timetuple02(self): self.assertEqual(dt28tz2.timetuple()[:8], (2010, 3, 28, 12, 0, 0, 6, 87)) diff --git a/python-stdlib/fnmatch/test_fnmatch.py b/python-stdlib/fnmatch/test_fnmatch.py index 4eaeec63b..97ef8fff7 100644 --- a/python-stdlib/fnmatch/test_fnmatch.py +++ b/python-stdlib/fnmatch/test_fnmatch.py @@ -1,6 +1,5 @@ """Test cases for the fnmatch module.""" -from test import support import unittest from fnmatch import fnmatch, fnmatchcase, translate, filter @@ -79,11 +78,3 @@ def test_translate(self): class FilterTestCase(unittest.TestCase): def test_filter(self): self.assertEqual(filter(["a", "b"], "a"), ["a"]) - - -def main(): - support.run_unittest(FnmatchTestCase, TranslateTestCase, FilterTestCase) - - -if __name__ == "__main__": - main() diff --git a/python-stdlib/gzip/gzip.py b/python-stdlib/gzip/gzip.py index c4473becb..12bfb1ff5 100644 --- a/python-stdlib/gzip/gzip.py +++ b/python-stdlib/gzip/gzip.py @@ -3,15 +3,15 @@ _WBITS = const(15) -import io, deflate +import builtins, io, deflate def GzipFile(fileobj): return deflate.DeflateIO(fileobj, deflate.GZIP, _WBITS) -def open(filename, mode): - return deflate.DeflateIO(open(filename, mode), deflate.GZIP, _WBITS, True) +def open(filename, mode="rb"): + return deflate.DeflateIO(builtins.open(filename, mode), deflate.GZIP, _WBITS, True) if hasattr(deflate.DeflateIO, "write"): diff --git a/python-stdlib/gzip/manifest.py b/python-stdlib/gzip/manifest.py index 006b538c5..c422b2965 100644 --- a/python-stdlib/gzip/manifest.py +++ b/python-stdlib/gzip/manifest.py @@ -1,3 +1,3 @@ -metadata(version="1.0.0") +metadata(version="1.0.1") module("gzip.py") diff --git a/python-stdlib/hashlib/tests/test_sha256.py b/python-stdlib/hashlib/tests/test_sha256.py index 024821b06..a311a8cc9 100644 --- a/python-stdlib/hashlib/tests/test_sha256.py +++ b/python-stdlib/hashlib/tests/test_sha256.py @@ -1,3 +1,7 @@ +# Prevent importing any built-in hashes, so this test tests only the pure Python hashes. +import sys +sys.modules['uhashlib'] = sys + import unittest from hashlib import sha256 diff --git a/python-stdlib/hmac/hmac.py b/python-stdlib/hmac/hmac.py index 28631042f..dbbdd4718 100644 --- a/python-stdlib/hmac/hmac.py +++ b/python-stdlib/hmac/hmac.py @@ -17,7 +17,7 @@ def __init__(self, key, msg=None, digestmod=None): make_hash = digestmod # A elif isinstance(digestmod, str): # A hash name suitable for hashlib.new(). - make_hash = lambda d=b"": hashlib.new(digestmod, d) # B + make_hash = lambda d=b"": getattr(hashlib, digestmod)(d) else: # A module supporting PEP 247. make_hash = digestmod.new # C diff --git a/python-stdlib/hmac/manifest.py b/python-stdlib/hmac/manifest.py index 28d78988d..ff0a62f08 100644 --- a/python-stdlib/hmac/manifest.py +++ b/python-stdlib/hmac/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.4.3") +metadata(version="3.4.4") module("hmac.py") diff --git a/python-stdlib/hmac/test_hmac.py b/python-stdlib/hmac/test_hmac.py index d155dd6a2..1cfcf4e37 100644 --- a/python-stdlib/hmac/test_hmac.py +++ b/python-stdlib/hmac/test_hmac.py @@ -8,7 +8,7 @@ msg = b"zlutoucky kun upel dabelske ody" -dig = hmac.new(b"1234567890", msg=msg, digestmod=hashlib.sha256).hexdigest() +dig = hmac.new(b"1234567890", msg=msg, digestmod="sha256").hexdigest() print("c735e751e36b08fb01e25794bdb15e7289b82aecdb652c8f4f72f307b39dad39") print(dig) diff --git a/python-stdlib/json/manifest.py b/python-stdlib/json/manifest.py deleted file mode 100644 index 87999e5f1..000000000 --- a/python-stdlib/json/manifest.py +++ /dev/null @@ -1,3 +0,0 @@ -metadata(version="0.1.0") - -package("json") diff --git a/python-stdlib/logging/logging.py b/python-stdlib/logging/logging.py index d17e42c4f..f4874df7d 100644 --- a/python-stdlib/logging/logging.py +++ b/python-stdlib/logging/logging.py @@ -58,6 +58,7 @@ def format(self, record): class StreamHandler(Handler): def __init__(self, stream=None): + super().__init__() self.stream = _stream if stream is None else stream self.terminator = "\n" diff --git a/python-stdlib/logging/manifest.py b/python-stdlib/logging/manifest.py index daf5d9c94..d9f0ee886 100644 --- a/python-stdlib/logging/manifest.py +++ b/python-stdlib/logging/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.6.0") +metadata(version="0.6.1") module("logging.py") diff --git a/python-stdlib/pathlib/pathlib.py b/python-stdlib/pathlib/pathlib.py index d01d81d32..e0f961373 100644 --- a/python-stdlib/pathlib/pathlib.py +++ b/python-stdlib/pathlib/pathlib.py @@ -47,6 +47,9 @@ def __init__(self, *segments): def __truediv__(self, other): return Path(self._path, str(other)) + def __rtruediv__(self, other): + return Path(other, self._path) + def __repr__(self): return f'{type(self).__name__}("{self._path}")' diff --git a/python-stdlib/pathlib/tests/test_pathlib.py b/python-stdlib/pathlib/tests/test_pathlib.py index c52cd9705..e632e1242 100644 --- a/python-stdlib/pathlib/tests/test_pathlib.py +++ b/python-stdlib/pathlib/tests/test_pathlib.py @@ -322,3 +322,14 @@ def test_with_suffix(self): self.assertTrue(Path("foo/test").with_suffix(".tar") == Path("foo/test.tar")) self.assertTrue(Path("foo/bar.bin").with_suffix(".txt") == Path("foo/bar.txt")) self.assertTrue(Path("bar.txt").with_suffix("") == Path("bar")) + + def test_rtruediv(self): + """Works as of micropython ea7031f""" + res = "foo" / Path("bar") + self.assertTrue(res == Path("foo/bar")) + + def test_rtruediv_inplace(self): + """Works as of micropython ea7031f""" + res = "foo" + res /= Path("bar") + self.assertTrue(res == Path("foo/bar")) diff --git a/python-stdlib/pkg_resources/pkg_resources.py b/python-stdlib/pkg_resources/pkg_resources.py index cd3e0fe96..d69cb0577 100644 --- a/python-stdlib/pkg_resources/pkg_resources.py +++ b/python-stdlib/pkg_resources/pkg_resources.py @@ -1,4 +1,4 @@ -import uio +import io c = {} @@ -18,11 +18,11 @@ def resource_stream(package, resource): else: d = "." # if d[0] != "/": - # import uos - # d = uos.getcwd() + "/" + d + # import os + # d = os.getcwd() + "/" + d c[package] = d + "/" p = c[package] if isinstance(p, dict): - return uio.BytesIO(p[resource]) + return io.BytesIO(p[resource]) return open(p + resource, "rb") diff --git a/python-stdlib/quopri/test_quopri.py b/python-stdlib/quopri/test_quopri.py index 5655dd8b0..b87e54842 100644 --- a/python-stdlib/quopri/test_quopri.py +++ b/python-stdlib/quopri/test_quopri.py @@ -1,7 +1,6 @@ -from test import support import unittest -import sys, os, io, subprocess +import sys, os, io import quopri @@ -193,7 +192,8 @@ def test_decode_header(self): for p, e in self.HSTRINGS: self.assertEqual(quopri.decodestring(e, header=True), p) - def _test_scriptencode(self): + @unittest.skip("requires subprocess") + def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen( [sys.executable, "-mquopri"], stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -210,7 +210,8 @@ def _test_scriptencode(self): self.assertEqual(cout[i], e[i]) self.assertEqual(cout, e) - def _test_scriptdecode(self): + @unittest.skip("requires subprocess") + def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen( [sys.executable, "-mquopri", "-d"], stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -220,11 +221,3 @@ def _test_scriptdecode(self): cout = cout.decode("latin-1") p = p.decode("latin-1") self.assertEqual(cout.splitlines(), p.splitlines()) - - -def test_main(): - support.run_unittest(QuopriTestCase) - - -if __name__ == "__main__": - test_main() diff --git a/python-stdlib/ssl/manifest.py b/python-stdlib/ssl/manifest.py index 1dae2f6e7..a99523071 100644 --- a/python-stdlib/ssl/manifest.py +++ b/python-stdlib/ssl/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.0") +metadata(version="0.2.1") -module("ssl.py") +module("ssl.py", opt=3) diff --git a/python-stdlib/ssl/ssl.py b/python-stdlib/ssl/ssl.py index 9262f5fb5..c61904be7 100644 --- a/python-stdlib/ssl/ssl.py +++ b/python-stdlib/ssl/ssl.py @@ -1,36 +1,65 @@ -from ussl import * -import ussl as _ussl +import tls +from tls import * -# Constants -for sym in "CERT_NONE", "CERT_OPTIONAL", "CERT_REQUIRED": - if sym not in globals(): - globals()[sym] = object() + +class SSLContext: + def __init__(self, *args): + self._context = tls.SSLContext(*args) + self._context.verify_mode = CERT_NONE + + @property + def verify_mode(self): + return self._context.verify_mode + + @verify_mode.setter + def verify_mode(self, val): + self._context.verify_mode = val + + def load_cert_chain(self, certfile, keyfile): + if isinstance(certfile, str): + with open(certfile, "rb") as f: + certfile = f.read() + if isinstance(keyfile, str): + with open(keyfile, "rb") as f: + keyfile = f.read() + self._context.load_cert_chain(certfile, keyfile) + + def load_verify_locations(self, cafile=None, cadata=None): + if cafile: + with open(cafile, "rb") as f: + cadata = f.read() + self._context.load_verify_locations(cadata) + + def wrap_socket( + self, sock, server_side=False, do_handshake_on_connect=True, server_hostname=None + ): + return self._context.wrap_socket( + sock, + server_side=server_side, + do_handshake_on_connect=do_handshake_on_connect, + server_hostname=server_hostname, + ) def wrap_socket( sock, - keyfile=None, - certfile=None, server_side=False, + key=None, + cert=None, cert_reqs=CERT_NONE, - *, - ca_certs=None, - server_hostname=None + cadata=None, + server_hostname=None, + do_handshake=True, ): - # TODO: More arguments accepted by CPython could also be handled here. - # That would allow us to accept ca_certs as a positional argument, which - # we should. - kw = {} - if keyfile is not None: - kw["keyfile"] = keyfile - if certfile is not None: - kw["certfile"] = certfile - if server_side is not False: - kw["server_side"] = server_side - if cert_reqs is not CERT_NONE: - kw["cert_reqs"] = cert_reqs - if ca_certs is not None: - kw["ca_certs"] = ca_certs - if server_hostname is not None: - kw["server_hostname"] = server_hostname - return _ussl.wrap_socket(sock, **kw) + con = SSLContext(PROTOCOL_TLS_SERVER if server_side else PROTOCOL_TLS_CLIENT) + if cert or key: + con.load_cert_chain(cert, key) + if cadata: + con.load_verify_locations(cadata=cadata) + con.verify_mode = cert_reqs + return con.wrap_socket( + sock, + server_side=server_side, + do_handshake_on_connect=do_handshake, + server_hostname=server_hostname, + ) diff --git a/python-stdlib/tarfile-write/manifest.py b/python-stdlib/tarfile-write/manifest.py index 248f7da60..bc4f37741 100644 --- a/python-stdlib/tarfile-write/manifest.py +++ b/python-stdlib/tarfile-write/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Adds write (create/append) support to tarfile.", version="0.1.1") +metadata(description="Adds write (create/append) support to tarfile.", version="0.1.2") require("tarfile") package("tarfile") diff --git a/python-stdlib/tarfile-write/tarfile/write.py b/python-stdlib/tarfile-write/tarfile/write.py index 062b8ae6b..527b3317b 100644 --- a/python-stdlib/tarfile-write/tarfile/write.py +++ b/python-stdlib/tarfile-write/tarfile/write.py @@ -67,7 +67,7 @@ def addfile(self, tarinfo, fileobj=None): name += "/" hdr = uctypes.struct(uctypes.addressof(buf), _TAR_HEADER, uctypes.LITTLE_ENDIAN) hdr.name[:] = name.encode("utf-8")[:100] - hdr.mode[:] = b"%07o\0" % (tarinfo.mode & 0o7777) + hdr.mode[:] = b"%07o\0" % ((0o755 if tarinfo.isdir() else 0o644) & 0o7777) hdr.uid[:] = b"%07o\0" % tarinfo.uid hdr.gid[:] = b"%07o\0" % tarinfo.gid hdr.size[:] = b"%011o\0" % size @@ -96,9 +96,10 @@ def addfile(self, tarinfo, fileobj=None): def add(self, name, recursive=True): from . import TarInfo - tarinfo = TarInfo(name) try: stat = os.stat(name) + res_name = (name + '/') if (stat[0] & 0xf000) == 0x4000 else name + tarinfo = TarInfo(res_name) tarinfo.mode = stat[0] tarinfo.uid = stat[4] tarinfo.gid = stat[5] diff --git a/python-stdlib/unittest-discover/manifest.py b/python-stdlib/unittest-discover/manifest.py index 14bec5201..5610f41e2 100644 --- a/python-stdlib/unittest-discover/manifest.py +++ b/python-stdlib/unittest-discover/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.1.2") +metadata(version="0.1.3") require("argparse") require("fnmatch") diff --git a/python-stdlib/unittest-discover/tests/sub/sub.py b/python-stdlib/unittest-discover/tests/sub/sub.py new file mode 100644 index 000000000..b6614dd63 --- /dev/null +++ b/python-stdlib/unittest-discover/tests/sub/sub.py @@ -0,0 +1 @@ +imported = True diff --git a/python-stdlib/unittest-discover/tests/sub/test_module_import.py b/python-stdlib/unittest-discover/tests/sub/test_module_import.py new file mode 100644 index 000000000..5c6404d6f --- /dev/null +++ b/python-stdlib/unittest-discover/tests/sub/test_module_import.py @@ -0,0 +1,13 @@ +import sys +import unittest + + +class TestModuleImport(unittest.TestCase): + def test_ModuleImportPath(self): + try: + from sub.sub import imported + assert imported + except ImportError: + print("This test is intended to be run with unittest discover" + "from the unittest-discover/tests dir. sys.path:", sys.path) + raise diff --git a/python-stdlib/unittest-discover/unittest/__main__.py b/python-stdlib/unittest-discover/unittest/__main__.py index 8eb173a22..09dfd03b9 100644 --- a/python-stdlib/unittest-discover/unittest/__main__.py +++ b/python-stdlib/unittest-discover/unittest/__main__.py @@ -6,7 +6,12 @@ from fnmatch import fnmatch from micropython import const -from unittest import TestRunner, TestResult, TestSuite +try: + from unittest import TestRunner, TestResult, TestSuite +except ImportError: + print("Error: This must be used from an installed copy of unittest-discover which will" + " also install base unittest module.") + raise # Run a single test in a clean environment. @@ -14,11 +19,11 @@ def _run_test_module(runner: TestRunner, module_name: str, *extra_paths: list[st module_snapshot = {k: v for k, v in sys.modules.items()} path_snapshot = sys.path[:] try: - for path in reversed(extra_paths): + for path in extra_paths: if path: sys.path.insert(0, path) - module = __import__(module_name) + module = __import__(module_name, None, None, module_name) suite = TestSuite(module_name) suite._load_module(module) return runner.run(suite) @@ -36,16 +41,18 @@ def _run_all_in_dir(runner: TestRunner, path: str, pattern: str, top: str): for fname, ftype, *_ in os.ilistdir(path): if fname in ("..", "."): continue + fpath = "/".join((path, fname)) if ftype == _DIR_TYPE: result += _run_all_in_dir( runner=runner, - path="/".join((path, fname)), + path=fpath, pattern=pattern, top=top, ) if fnmatch(fname, pattern): - module_name = fname.rsplit(".", 1)[0] - result += _run_test_module(runner, module_name, path, top) + module_path = fpath.rsplit(".", 1)[0] # remove ext + module_path = module_path.replace("/", ".").strip(".") + result += _run_test_module(runner, module_path, top) return result diff --git a/tools/build.py b/tools/build.py index ca664175f..442cf2121 100755 --- a/tools/build.py +++ b/tools/build.py @@ -64,7 +64,7 @@ # index.json is: # { -# "v": 1, <-- file format version +# "v": 2, <-- file format version # "updated": , # "packages": { # { @@ -78,7 +78,9 @@ # "7": ["0.2", "0.3", "0.4"], # ... <-- Other bytecode versions # "py": ["0.1", "0.2", "0.3", "0.4"] -# } +# }, +# // The following entries were added in file format version 2. +# path: "micropython/bluetooth/aioble", # }, # ... # } @@ -122,7 +124,7 @@ import time -_JSON_VERSION_INDEX = 1 +_JSON_VERSION_INDEX = 2 _JSON_VERSION_PACKAGE = 1 @@ -268,7 +270,7 @@ def _copy_as_py( # Update to the latest metadata, and add any new versions to the package in # the index json. -def _update_index_package_metadata(index_package_json, metadata, mpy_version): +def _update_index_package_metadata(index_package_json, metadata, mpy_version, package_path): index_package_json["version"] = metadata.version or "" index_package_json["author"] = "" # TODO: Make manifestfile.py capture this. index_package_json["description"] = metadata.description or "" @@ -283,6 +285,9 @@ def _update_index_package_metadata(index_package_json, metadata, mpy_version): print(" New version {}={}".format(v, metadata.version)) index_package_json["versions"][v].append(metadata.version) + # The following entries were added in file format version 2. + index_package_json["path"] = package_path + def build(output_path, hash_prefix_len, mpy_cross_path): import manifestfile @@ -318,7 +323,8 @@ def build(output_path, hash_prefix_len, mpy_cross_path): for lib_dir in lib_dirs: for manifest_path in glob.glob(os.path.join(lib_dir, "**", "manifest.py"), recursive=True): - print("{}".format(os.path.dirname(manifest_path))) + package_path = os.path.dirname(manifest_path) + print("{}".format(package_path)) # .../foo/manifest.py -> foo package_name = os.path.basename(os.path.dirname(manifest_path)) @@ -342,7 +348,9 @@ def build(output_path, hash_prefix_len, mpy_cross_path): } index_json["packages"].append(index_package_json) - _update_index_package_metadata(index_package_json, manifest.metadata(), mpy_version) + _update_index_package_metadata( + index_package_json, manifest.metadata(), mpy_version, package_path + ) # This is the package json that mip/mpremote downloads. mpy_package_json = { diff --git a/tools/ci.sh b/tools/ci.sh index 730034efb..07b27d13c 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -1,5 +1,7 @@ #!/bin/bash +CP=/bin/cp + ######################################################################################## # commit formatting @@ -12,6 +14,95 @@ function ci_commit_formatting_run { tools/verifygitlog.py -v upstream/master..HEAD --no-merges } +######################################################################################## +# package tests + +MICROPYTHON=/tmp/micropython/ports/unix/build-standard/micropython + +function ci_package_tests_setup_micropython { + git clone https://github.com/micropython/micropython.git /tmp/micropython + + # build mpy-cross and micropython (use -O0 to speed up the build) + make -C /tmp/micropython/mpy-cross -j CFLAGS_EXTRA=-O0 + make -C /tmp/micropython/ports/unix submodules + make -C /tmp/micropython/ports/unix -j CFLAGS_EXTRA=-O0 +} + +function ci_package_tests_setup_lib { + mkdir -p ~/.micropython/lib + $CP micropython/ucontextlib/ucontextlib.py ~/.micropython/lib/ + $CP python-stdlib/fnmatch/fnmatch.py ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-core/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha224/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha256/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha384/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha512/hashlib ~/.micropython/lib/ + $CP python-stdlib/shutil/shutil.py ~/.micropython/lib/ + $CP python-stdlib/tempfile/tempfile.py ~/.micropython/lib/ + $CP -r python-stdlib/unittest/unittest ~/.micropython/lib/ + $CP -r python-stdlib/unittest-discover/unittest ~/.micropython/lib/ + $CP unix-ffi/ffilib/ffilib.py ~/.micropython/lib/ + tree ~/.micropython +} + +function ci_package_tests_run { + for test in \ + micropython/drivers/storage/sdcard/sdtest.py \ + micropython/xmltok/test_xmltok.py \ + python-ecosys/requests/test_requests.py \ + python-stdlib/argparse/test_argparse.py \ + python-stdlib/base64/test_base64.py \ + python-stdlib/binascii/test_binascii.py \ + python-stdlib/collections-defaultdict/test_defaultdict.py \ + python-stdlib/functools/test_partial.py \ + python-stdlib/functools/test_reduce.py \ + python-stdlib/heapq/test_heapq.py \ + python-stdlib/hmac/test_hmac.py \ + python-stdlib/itertools/test_itertools.py \ + python-stdlib/operator/test_operator.py \ + python-stdlib/os-path/test_path.py \ + python-stdlib/pickle/test_pickle.py \ + python-stdlib/string/test_translate.py \ + unix-ffi/gettext/test_gettext.py \ + unix-ffi/pwd/test_getpwnam.py \ + unix-ffi/re/test_re.py \ + unix-ffi/sqlite3/test_sqlite3.py \ + unix-ffi/sqlite3/test_sqlite3_2.py \ + unix-ffi/sqlite3/test_sqlite3_3.py \ + unix-ffi/time/test_strftime.py \ + ; do + echo "Running test $test" + (cd `dirname $test` && $MICROPYTHON `basename $test`) + if [ $? -ne 0 ]; then + false # make this function return an error code + return + fi + done + + for path in \ + micropython/ucontextlib \ + python-stdlib/contextlib \ + python-stdlib/datetime \ + python-stdlib/fnmatch \ + python-stdlib/hashlib \ + python-stdlib/pathlib \ + python-stdlib/quopri \ + python-stdlib/shutil \ + python-stdlib/tempfile \ + python-stdlib/time \ + python-stdlib/unittest-discover/tests \ + ; do + (cd $path && $MICROPYTHON -m unittest) + if [ $? -ne 0 ]; then false; return; fi + done + + (cd micropython/usb/usb-device && $MICROPYTHON -m tests.test_core_buffer) + if [ $? -ne 0 ]; then false; return; fi + + (cd python-ecosys/cbor2 && $MICROPYTHON -m examples.cbor_test) + if [ $? -ne 0 ]; then false; return; fi +} + ######################################################################################## # build packages @@ -30,7 +121,11 @@ function ci_build_packages_check_manifest { for file in $(find -name manifest.py); do echo "##################################################" echo "# Testing $file" - python3 /tmp/micropython/tools/manifestfile.py --lib . --compile $file + extra_args= + if [[ "$file" =~ "/unix-ffi/" ]]; then + extra_args="--unix-ffi" + fi + python3 /tmp/micropython/tools/manifestfile.py $extra_args --lib . --compile $file done } diff --git a/unix-ffi/README.md b/unix-ffi/README.md index d6b9417d8..6ea05d65a 100644 --- a/unix-ffi/README.md +++ b/unix-ffi/README.md @@ -19,9 +19,13 @@ replacement for CPython. ### Usage -To use a unix-specific library, pass `unix_ffi=True` to `require()` in your -manifest file. +To use a unix-specific library, a manifest file must add the `unix-ffi` +library to the library search path using `add_library()`: ```py -require("os", unix_ffi=True) # Use the unix-ffi version instead of python-stdlib. +add_library("unix-ffi", "$(MPY_LIB_DIR)/unix-ffi", prepend=True) ``` + +Prepending the `unix-ffi` library to the path will make it so that the +`unix-ffi` version of a package will be preferred if that package appears in +both `unix-ffi` and another library (eg `python-stdlib`). diff --git a/unix-ffi/_markupbase/manifest.py b/unix-ffi/_markupbase/manifest.py index ec576c28f..9cbf52bb0 100644 --- a/unix-ffi/_markupbase/manifest.py +++ b/unix-ffi/_markupbase/manifest.py @@ -1,5 +1,5 @@ metadata(version="3.3.4") -require("re", unix_ffi=True) +require("re") module("_markupbase.py") diff --git a/unix-ffi/email.charset/manifest.py b/unix-ffi/email.charset/manifest.py index 31d70cece..7e6dd7936 100644 --- a/unix-ffi/email.charset/manifest.py +++ b/unix-ffi/email.charset/manifest.py @@ -1,7 +1,7 @@ metadata(version="0.5.1") require("functools") -require("email.encoders", unix_ffi=True) -require("email.errors", unix_ffi=True) +require("email.encoders") +require("email.errors") package("email") diff --git a/unix-ffi/email.encoders/manifest.py b/unix-ffi/email.encoders/manifest.py index e1e2090c9..a3e735d8c 100644 --- a/unix-ffi/email.encoders/manifest.py +++ b/unix-ffi/email.encoders/manifest.py @@ -3,7 +3,7 @@ require("base64") require("binascii") require("quopri") -require("re", unix_ffi=True) +require("re") require("string") package("email") diff --git a/unix-ffi/email.feedparser/manifest.py b/unix-ffi/email.feedparser/manifest.py index 4ea80e302..31c34ba91 100644 --- a/unix-ffi/email.feedparser/manifest.py +++ b/unix-ffi/email.feedparser/manifest.py @@ -1,8 +1,8 @@ metadata(version="0.5.1") -require("re", unix_ffi=True) -require("email.errors", unix_ffi=True) -require("email.message", unix_ffi=True) -require("email.internal", unix_ffi=True) +require("re") +require("email.errors") +require("email.message") +require("email.internal") package("email") diff --git a/unix-ffi/email.header/manifest.py b/unix-ffi/email.header/manifest.py index 65b017b50..0be7e85c2 100644 --- a/unix-ffi/email.header/manifest.py +++ b/unix-ffi/email.header/manifest.py @@ -1,9 +1,9 @@ metadata(version="0.5.2") -require("re", unix_ffi=True) +require("re") require("binascii") -require("email.encoders", unix_ffi=True) -require("email.errors", unix_ffi=True) -require("email.charset", unix_ffi=True) +require("email.encoders") +require("email.errors") +require("email.charset") package("email") diff --git a/unix-ffi/email.internal/manifest.py b/unix-ffi/email.internal/manifest.py index 4aff6d2c5..88acb2c01 100644 --- a/unix-ffi/email.internal/manifest.py +++ b/unix-ffi/email.internal/manifest.py @@ -1,15 +1,15 @@ metadata(version="0.5.1") -require("re", unix_ffi=True) +require("re") require("base64") require("binascii") require("functools") require("string") # require("calendar") TODO require("abc") -require("email.errors", unix_ffi=True) -require("email.header", unix_ffi=True) -require("email.charset", unix_ffi=True) -require("email.utils", unix_ffi=True) +require("email.errors") +require("email.header") +require("email.charset") +require("email.utils") package("email") diff --git a/unix-ffi/email.message/manifest.py b/unix-ffi/email.message/manifest.py index 7b75ee7ac..d1849de35 100644 --- a/unix-ffi/email.message/manifest.py +++ b/unix-ffi/email.message/manifest.py @@ -1,11 +1,11 @@ metadata(version="0.5.3") -require("re", unix_ffi=True) +require("re") require("uu") require("base64") require("binascii") -require("email.utils", unix_ffi=True) -require("email.errors", unix_ffi=True) -require("email.charset", unix_ffi=True) +require("email.utils") +require("email.errors") +require("email.charset") package("email") diff --git a/unix-ffi/email.parser/manifest.py b/unix-ffi/email.parser/manifest.py index ebe662111..dd8aacde8 100644 --- a/unix-ffi/email.parser/manifest.py +++ b/unix-ffi/email.parser/manifest.py @@ -1,8 +1,8 @@ metadata(version="0.5.1") require("warnings") -require("email.feedparser", unix_ffi=True) -require("email.message", unix_ffi=True) -require("email.internal", unix_ffi=True) +require("email.feedparser") +require("email.message") +require("email.internal") package("email") diff --git a/unix-ffi/email.utils/manifest.py b/unix-ffi/email.utils/manifest.py index 20b21b406..a7208536d 100644 --- a/unix-ffi/email.utils/manifest.py +++ b/unix-ffi/email.utils/manifest.py @@ -1,13 +1,13 @@ metadata(version="3.3.4") -require("os", unix_ffi=True) -require("re", unix_ffi=True) +require("os") +require("re") require("base64") require("random") require("datetime") -require("urllib.parse", unix_ffi=True) +require("urllib.parse") require("warnings") require("quopri") -require("email.charset", unix_ffi=True) +require("email.charset") package("email") diff --git a/unix-ffi/fcntl/manifest.py b/unix-ffi/fcntl/manifest.py index a0e9d9592..e572a58e8 100644 --- a/unix-ffi/fcntl/manifest.py +++ b/unix-ffi/fcntl/manifest.py @@ -2,6 +2,6 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("fcntl.py") diff --git a/unix-ffi/getopt/manifest.py b/unix-ffi/getopt/manifest.py index ae28ffd7f..cde6c4c09 100644 --- a/unix-ffi/getopt/manifest.py +++ b/unix-ffi/getopt/manifest.py @@ -1,5 +1,5 @@ metadata(version="3.3.4") -require("os", unix_ffi=True) +require("os") module("getopt.py") diff --git a/unix-ffi/gettext/manifest.py b/unix-ffi/gettext/manifest.py index 94b58b2e4..fe40b01b6 100644 --- a/unix-ffi/gettext/manifest.py +++ b/unix-ffi/gettext/manifest.py @@ -2,6 +2,6 @@ # Originally written by Riccardo Magliocchetti. -require("ffilib", unix_ffi=True) +require("ffilib") module("gettext.py") diff --git a/unix-ffi/glob/manifest.py b/unix-ffi/glob/manifest.py index 622289bca..2d2fab31c 100644 --- a/unix-ffi/glob/manifest.py +++ b/unix-ffi/glob/manifest.py @@ -1,8 +1,8 @@ metadata(version="0.5.2") -require("os", unix_ffi=True) +require("os") require("os-path") -require("re", unix_ffi=True) +require("re") require("fnmatch") module("glob.py") diff --git a/unix-ffi/html.parser/manifest.py b/unix-ffi/html.parser/manifest.py index a0a5bc4f4..3f29bbceb 100644 --- a/unix-ffi/html.parser/manifest.py +++ b/unix-ffi/html.parser/manifest.py @@ -1,8 +1,8 @@ metadata(version="3.3.4") -require("_markupbase", unix_ffi=True) +require("_markupbase") require("warnings") -require("html.entities", unix_ffi=True) -require("re", unix_ffi=True) +require("html.entities") +require("re") package("html") diff --git a/unix-ffi/http.client/manifest.py b/unix-ffi/http.client/manifest.py index be0c9ef36..add274422 100644 --- a/unix-ffi/http.client/manifest.py +++ b/unix-ffi/http.client/manifest.py @@ -1,10 +1,10 @@ metadata(version="0.5.1") -require("email.parser", unix_ffi=True) -require("email.message", unix_ffi=True) -require("socket", unix_ffi=True) +require("email.parser") +require("email.message") +require("socket") require("collections") -require("urllib.parse", unix_ffi=True) +require("urllib.parse") require("warnings") package("http") diff --git a/python-stdlib/json/json/__init__.py b/unix-ffi/json/json/__init__.py similarity index 100% rename from python-stdlib/json/json/__init__.py rename to unix-ffi/json/json/__init__.py diff --git a/python-stdlib/json/json/decoder.py b/unix-ffi/json/json/decoder.py similarity index 100% rename from python-stdlib/json/json/decoder.py rename to unix-ffi/json/json/decoder.py diff --git a/python-stdlib/json/json/encoder.py b/unix-ffi/json/json/encoder.py similarity index 100% rename from python-stdlib/json/json/encoder.py rename to unix-ffi/json/json/encoder.py diff --git a/python-stdlib/json/json/scanner.py b/unix-ffi/json/json/scanner.py similarity index 100% rename from python-stdlib/json/json/scanner.py rename to unix-ffi/json/json/scanner.py diff --git a/python-stdlib/json/json/tool.py b/unix-ffi/json/json/tool.py similarity index 100% rename from python-stdlib/json/json/tool.py rename to unix-ffi/json/json/tool.py diff --git a/unix-ffi/json/manifest.py b/unix-ffi/json/manifest.py new file mode 100644 index 000000000..9267719f1 --- /dev/null +++ b/unix-ffi/json/manifest.py @@ -0,0 +1,4 @@ +metadata(version="0.2.0") + +require("re") +package("json") diff --git a/python-stdlib/json/test_json.py b/unix-ffi/json/test_json.py similarity index 100% rename from python-stdlib/json/test_json.py rename to unix-ffi/json/test_json.py diff --git a/unix-ffi/machine/example_timer.py b/unix-ffi/machine/example_timer.py index a0d44110f..550d68cd3 100644 --- a/unix-ffi/machine/example_timer.py +++ b/unix-ffi/machine/example_timer.py @@ -1,4 +1,4 @@ -import utime +import time from machine import Timer @@ -7,5 +7,5 @@ t1.callback(lambda t: print(t, "tick1")) t2.callback(lambda t: print(t, "tick2")) -utime.sleep(3) +time.sleep(3) print("done") diff --git a/unix-ffi/machine/machine/timer.py b/unix-ffi/machine/machine/timer.py index 1aa53f936..3f371142c 100644 --- a/unix-ffi/machine/machine/timer.py +++ b/unix-ffi/machine/machine/timer.py @@ -1,9 +1,7 @@ import ffilib import uctypes import array -import uos import os -import utime from signal import * libc = ffilib.libc() diff --git a/unix-ffi/machine/manifest.py b/unix-ffi/machine/manifest.py index 9c1f34775..c0e40764d 100644 --- a/unix-ffi/machine/manifest.py +++ b/unix-ffi/machine/manifest.py @@ -2,8 +2,8 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) -require("os", unix_ffi=True) -require("signal", unix_ffi=True) +require("ffilib") +require("os") +require("signal") package("machine") diff --git a/unix-ffi/multiprocessing/manifest.py b/unix-ffi/multiprocessing/manifest.py index 68f2bca08..d6b32411d 100644 --- a/unix-ffi/multiprocessing/manifest.py +++ b/unix-ffi/multiprocessing/manifest.py @@ -2,8 +2,8 @@ # Originally written by Paul Sokolovsky. -require("os", unix_ffi=True) -require("select", unix_ffi=True) +require("os") +require("select") require("pickle") module("multiprocessing.py") diff --git a/unix-ffi/os/manifest.py b/unix-ffi/os/manifest.py index 0dce28e0b..e4bc100a2 100644 --- a/unix-ffi/os/manifest.py +++ b/unix-ffi/os/manifest.py @@ -2,7 +2,7 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") require("errno") require("stat") diff --git a/unix-ffi/os/os/__init__.py b/unix-ffi/os/os/__init__.py index 3cca078f9..6c87da892 100644 --- a/unix-ffi/os/os/__init__.py +++ b/unix-ffi/os/os/__init__.py @@ -1,5 +1,5 @@ import array -import ustruct as struct +import struct import errno as errno_ import stat as stat_ import ffilib diff --git a/unix-ffi/pwd/manifest.py b/unix-ffi/pwd/manifest.py index 49bfb403e..fd422aaeb 100644 --- a/unix-ffi/pwd/manifest.py +++ b/unix-ffi/pwd/manifest.py @@ -2,6 +2,6 @@ # Originally written by Riccardo Magliocchetti. -require("ffilib", unix_ffi=True) +require("ffilib") module("pwd.py") diff --git a/unix-ffi/pwd/pwd.py b/unix-ffi/pwd/pwd.py index 29ebe3416..561269ed2 100644 --- a/unix-ffi/pwd/pwd.py +++ b/unix-ffi/pwd/pwd.py @@ -1,8 +1,8 @@ import ffilib import uctypes -import ustruct +import struct -from ucollections import namedtuple +from collections import namedtuple libc = ffilib.libc() @@ -20,6 +20,6 @@ def getpwnam(user): if not passwd: raise KeyError("getpwnam(): name not found: {}".format(user)) passwd_fmt = "SSIISSS" - passwd = uctypes.bytes_at(passwd, ustruct.calcsize(passwd_fmt)) - passwd = ustruct.unpack(passwd_fmt, passwd) + passwd = uctypes.bytes_at(passwd, struct.calcsize(passwd_fmt)) + passwd = struct.unpack(passwd_fmt, passwd) return struct_passwd(*passwd) diff --git a/unix-ffi/pyusb/examples/lsusb.py b/unix-ffi/pyusb/examples/lsusb.py new file mode 100644 index 000000000..549043567 --- /dev/null +++ b/unix-ffi/pyusb/examples/lsusb.py @@ -0,0 +1,18 @@ +# Simple example to list attached USB devices. + +import usb.core + +for device in usb.core.find(find_all=True): + print("ID {:04x}:{:04x}".format(device.idVendor, device.idProduct)) + for cfg in device: + print( + " config numitf={} value={} attr={} power={}".format( + cfg.bNumInterfaces, cfg.bConfigurationValue, cfg.bmAttributes, cfg.bMaxPower + ) + ) + for itf in cfg: + print( + " interface class={} subclass={}".format( + itf.bInterfaceClass, itf.bInterfaceSubClass + ) + ) diff --git a/unix-ffi/pyusb/manifest.py b/unix-ffi/pyusb/manifest.py new file mode 100644 index 000000000..d60076255 --- /dev/null +++ b/unix-ffi/pyusb/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0", pypi="pyusb") + +package("usb") diff --git a/unix-ffi/pyusb/usb/__init__.py b/unix-ffi/pyusb/usb/__init__.py new file mode 100644 index 000000000..19afe623c --- /dev/null +++ b/unix-ffi/pyusb/usb/__init__.py @@ -0,0 +1,2 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George diff --git a/unix-ffi/pyusb/usb/control.py b/unix-ffi/pyusb/usb/control.py new file mode 100644 index 000000000..b03a89464 --- /dev/null +++ b/unix-ffi/pyusb/usb/control.py @@ -0,0 +1,10 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + + +def get_descriptor(dev, desc_size, desc_type, desc_index, wIndex=0): + wValue = desc_index | desc_type << 8 + d = dev.ctrl_transfer(0x80, 0x06, wValue, wIndex, desc_size) + if len(d) < 2: + raise Exception("invalid descriptor") + return d diff --git a/unix-ffi/pyusb/usb/core.py b/unix-ffi/pyusb/usb/core.py new file mode 100644 index 000000000..bfb0a028d --- /dev/null +++ b/unix-ffi/pyusb/usb/core.py @@ -0,0 +1,239 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + +import sys +import ffi +import uctypes + +if sys.maxsize >> 32: + UINTPTR_SIZE = 8 + UINTPTR = uctypes.UINT64 +else: + UINTPTR_SIZE = 4 + UINTPTR = uctypes.UINT32 + + +def _align_word(x): + return (x + UINTPTR_SIZE - 1) & ~(UINTPTR_SIZE - 1) + + +ptr_descriptor = (0 | uctypes.ARRAY, 1 | UINTPTR) + +libusb_device_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "bcdUSB": 2 | uctypes.UINT16, + "bDeviceClass": 4 | uctypes.UINT8, + "bDeviceSubClass": 5 | uctypes.UINT8, + "bDeviceProtocol": 6 | uctypes.UINT8, + "bMaxPacketSize0": 7 | uctypes.UINT8, + "idVendor": 8 | uctypes.UINT16, + "idProduct": 10 | uctypes.UINT16, + "bcdDevice": 12 | uctypes.UINT16, + "iManufacturer": 14 | uctypes.UINT8, + "iProduct": 15 | uctypes.UINT8, + "iSerialNumber": 16 | uctypes.UINT8, + "bNumConfigurations": 17 | uctypes.UINT8, +} + +libusb_config_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "wTotalLength": 2 | uctypes.UINT16, + "bNumInterfaces": 4 | uctypes.UINT8, + "bConfigurationValue": 5 | uctypes.UINT8, + "iConfiguration": 6 | uctypes.UINT8, + "bmAttributes": 7 | uctypes.UINT8, + "MaxPower": 8 | uctypes.UINT8, + "interface": _align_word(9) | UINTPTR, # array of libusb_interface + "extra": (_align_word(9) + UINTPTR_SIZE) | UINTPTR, + "extra_length": (_align_word(9) + 2 * UINTPTR_SIZE) | uctypes.INT, +} + +libusb_interface = { + "altsetting": 0 | UINTPTR, # array of libusb_interface_descriptor + "num_altsetting": UINTPTR_SIZE | uctypes.INT, +} + +libusb_interface_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "bInterfaceNumber": 2 | uctypes.UINT8, + "bAlternateSetting": 3 | uctypes.UINT8, + "bNumEndpoints": 4 | uctypes.UINT8, + "bInterfaceClass": 5 | uctypes.UINT8, + "bInterfaceSubClass": 6 | uctypes.UINT8, + "bInterfaceProtocol": 7 | uctypes.UINT8, + "iInterface": 8 | uctypes.UINT8, + "endpoint": _align_word(9) | UINTPTR, + "extra": (_align_word(9) + UINTPTR_SIZE) | UINTPTR, + "extra_length": (_align_word(9) + 2 * UINTPTR_SIZE) | uctypes.INT, +} + +libusb = ffi.open("libusb-1.0.so") +libusb_init = libusb.func("i", "libusb_init", "p") +libusb_exit = libusb.func("v", "libusb_exit", "p") +libusb_get_device_list = libusb.func("i", "libusb_get_device_list", "pp") # return is ssize_t +libusb_free_device_list = libusb.func("v", "libusb_free_device_list", "pi") +libusb_get_device_descriptor = libusb.func("i", "libusb_get_device_descriptor", "pp") +libusb_get_config_descriptor = libusb.func("i", "libusb_get_config_descriptor", "pBp") +libusb_free_config_descriptor = libusb.func("v", "libusb_free_config_descriptor", "p") +libusb_open = libusb.func("i", "libusb_open", "pp") +libusb_set_configuration = libusb.func("i", "libusb_set_configuration", "pi") +libusb_claim_interface = libusb.func("i", "libusb_claim_interface", "pi") +libusb_control_transfer = libusb.func("i", "libusb_control_transfer", "pBBHHpHI") + + +def _new(sdesc): + buf = bytearray(uctypes.sizeof(sdesc)) + s = uctypes.struct(uctypes.addressof(buf), sdesc) + return s + + +class Interface: + def __init__(self, descr): + # Public attributes. + self.bInterfaceClass = descr.bInterfaceClass + self.bInterfaceSubClass = descr.bInterfaceSubClass + self.iInterface = descr.iInterface + self.extra_descriptors = uctypes.bytes_at(descr.extra, descr.extra_length) + + +class Configuration: + def __init__(self, dev, cfg_idx): + cfgs = _new(ptr_descriptor) + if libusb_get_config_descriptor(dev._dev, cfg_idx, cfgs) != 0: + libusb_exit(0) + raise Exception + descr = uctypes.struct(cfgs[0], libusb_config_descriptor) + + # Extract all needed info because descr is going to be free'd at the end. + self._itfs = [] + itf_array = uctypes.struct( + descr.interface, (0 | uctypes.ARRAY, descr.bNumInterfaces, libusb_interface) + ) + for i in range(descr.bNumInterfaces): + itf = itf_array[i] + alt_array = uctypes.struct( + itf.altsetting, + (0 | uctypes.ARRAY, itf.num_altsetting, libusb_interface_descriptor), + ) + for j in range(itf.num_altsetting): + alt = alt_array[j] + self._itfs.append(Interface(alt)) + + # Public attributes. + self.bNumInterfaces = descr.bNumInterfaces + self.bConfigurationValue = descr.bConfigurationValue + self.bmAttributes = descr.bmAttributes + self.bMaxPower = descr.MaxPower + self.extra_descriptors = uctypes.bytes_at(descr.extra, descr.extra_length) + + # Free descr memory in the driver. + libusb_free_config_descriptor(cfgs[0]) + + def __iter__(self): + return iter(self._itfs) + + +class Device: + _TIMEOUT_DEFAULT = 1000 + + def __init__(self, dev, descr): + self._dev = dev + self._num_cfg = descr.bNumConfigurations + self._handle = None + self._claim_itf = set() + + # Public attributes. + self.idVendor = descr.idVendor + self.idProduct = descr.idProduct + + def __iter__(self): + for i in range(self._num_cfg): + yield Configuration(self, i) + + def __getitem__(self, i): + return Configuration(self, i) + + def _open(self): + if self._handle is None: + # Open the USB device. + handle = _new(ptr_descriptor) + if libusb_open(self._dev, handle) != 0: + libusb_exit(0) + raise Exception + self._handle = handle[0] + + def _claim_interface(self, i): + if libusb_claim_interface(self._handle, i) != 0: + libusb_exit(0) + raise Exception + + def set_configuration(self): + # Select default configuration. + self._open() + cfg = Configuration(self, 0).bConfigurationValue + ret = libusb_set_configuration(self._handle, cfg) + if ret != 0: + libusb_exit(0) + raise Exception + + def ctrl_transfer( + self, bmRequestType, bRequest, wValue=0, wIndex=0, data_or_wLength=None, timeout=None + ): + if data_or_wLength is None: + l = 0 + data = bytes() + elif isinstance(data_or_wLength, int): + l = data_or_wLength + data = bytearray(l) + else: + l = len(data_or_wLength) + data = data_or_wLength + self._open() + if wIndex & 0xFF not in self._claim_itf: + self._claim_interface(wIndex & 0xFF) + self._claim_itf.add(wIndex & 0xFF) + if timeout is None: + timeout = self._TIMEOUT_DEFAULT + ret = libusb_control_transfer( + self._handle, bmRequestType, bRequest, wValue, wIndex, data, l, timeout * 1000 + ) + if ret < 0: + libusb_exit(0) + raise Exception + if isinstance(data_or_wLength, int): + return data + else: + return ret + + +def find(*, find_all=False, custom_match=None, idVendor=None, idProduct=None): + if libusb_init(0) < 0: + raise Exception + + devs = _new(ptr_descriptor) + count = libusb_get_device_list(0, devs) + if count < 0: + libusb_exit(0) + raise Exception + + dev_array = uctypes.struct(devs[0], (0 | uctypes.ARRAY, count | UINTPTR)) + descr = _new(libusb_device_descriptor) + devices = None + for i in range(count): + libusb_get_device_descriptor(dev_array[i], descr) + if idVendor and descr.idVendor != idVendor: + continue + if idProduct and descr.idProduct != idProduct: + continue + device = Device(dev_array[i], descr) + if custom_match and not custom_match(device): + continue + if not find_all: + return device + if not devices: + devices = [] + devices.append(device) + return devices diff --git a/unix-ffi/pyusb/usb/util.py b/unix-ffi/pyusb/usb/util.py new file mode 100644 index 000000000..04e4763e4 --- /dev/null +++ b/unix-ffi/pyusb/usb/util.py @@ -0,0 +1,16 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + +import usb.control + + +def claim_interface(device, interface): + device._claim_interface(interface) + + +def get_string(device, index): + bs = usb.control.get_descriptor(device, 254, 3, index, 0) + s = "" + for i in range(2, bs[0] & 0xFE, 2): + s += chr(bs[i] | bs[i + 1] << 8) + return s diff --git a/unix-ffi/re/manifest.py b/unix-ffi/re/manifest.py index cc52df47a..ca027317d 100644 --- a/unix-ffi/re/manifest.py +++ b/unix-ffi/re/manifest.py @@ -2,6 +2,6 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("re.py") diff --git a/unix-ffi/select/manifest.py b/unix-ffi/select/manifest.py index ef2778ed7..b9576de5e 100644 --- a/unix-ffi/select/manifest.py +++ b/unix-ffi/select/manifest.py @@ -2,7 +2,7 @@ # Originally written by Paul Sokolovsky. -require("os", unix_ffi=True) -require("ffilib", unix_ffi=True) +require("os") +require("ffilib") module("select.py") diff --git a/unix-ffi/select/select.py b/unix-ffi/select/select.py index eec9bfb81..9d514a31d 100644 --- a/unix-ffi/select/select.py +++ b/unix-ffi/select/select.py @@ -1,5 +1,5 @@ import ffi -import ustruct as struct +import struct import os import errno import ffilib diff --git a/unix-ffi/signal/manifest.py b/unix-ffi/signal/manifest.py index 913bbdc8c..cb23542cc 100644 --- a/unix-ffi/signal/manifest.py +++ b/unix-ffi/signal/manifest.py @@ -2,6 +2,6 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("signal.py") diff --git a/unix-ffi/sqlite3/manifest.py b/unix-ffi/sqlite3/manifest.py index e941e1ddd..5b04d71d3 100644 --- a/unix-ffi/sqlite3/manifest.py +++ b/unix-ffi/sqlite3/manifest.py @@ -1,7 +1,7 @@ -metadata(version="0.2.4") +metadata(version="0.3.0") # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("sqlite3.py") diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 0f00ff508..299f8247d 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -1,12 +1,21 @@ import sys import ffilib +import uctypes sq3 = ffilib.open("libsqlite3") +# int sqlite3_open( +# const char *filename, /* Database filename (UTF-8) */ +# sqlite3 **ppDb /* OUT: SQLite db handle */ +# ); sqlite3_open = sq3.func("i", "sqlite3_open", "sp") -# int sqlite3_close(sqlite3*); -sqlite3_close = sq3.func("i", "sqlite3_close", "p") +# int sqlite3_config(int, ...); +sqlite3_config = sq3.func("i", "sqlite3_config", "ii") +# int sqlite3_get_autocommit(sqlite3*); +sqlite3_get_autocommit = sq3.func("i", "sqlite3_get_autocommit", "p") +# int sqlite3_close_v2(sqlite3*); +sqlite3_close = sq3.func("i", "sqlite3_close_v2", "p") # int sqlite3_prepare( # sqlite3 *db, /* Database handle */ # const char *zSql, /* SQL statement, UTF-8 encoded */ @@ -14,7 +23,7 @@ # sqlite3_stmt **ppStmt, /* OUT: Statement handle */ # const char **pzTail /* OUT: Pointer to unused portion of zSql */ # ); -sqlite3_prepare = sq3.func("i", "sqlite3_prepare", "psipp") +sqlite3_prepare = sq3.func("i", "sqlite3_prepare_v2", "psipp") # int sqlite3_finalize(sqlite3_stmt *pStmt); sqlite3_finalize = sq3.func("i", "sqlite3_finalize", "p") # int sqlite3_step(sqlite3_stmt*); @@ -23,20 +32,17 @@ sqlite3_column_count = sq3.func("i", "sqlite3_column_count", "p") # int sqlite3_column_type(sqlite3_stmt*, int iCol); sqlite3_column_type = sq3.func("i", "sqlite3_column_type", "pi") +# int sqlite3_column_int(sqlite3_stmt*, int iCol); sqlite3_column_int = sq3.func("i", "sqlite3_column_int", "pi") -# using "d" return type gives wrong results +# double sqlite3_column_double(sqlite3_stmt*, int iCol); sqlite3_column_double = sq3.func("d", "sqlite3_column_double", "pi") +# const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); sqlite3_column_text = sq3.func("s", "sqlite3_column_text", "pi") # sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); -# TODO: should return long int -sqlite3_last_insert_rowid = sq3.func("i", "sqlite3_last_insert_rowid", "p") +sqlite3_last_insert_rowid = sq3.func("l", "sqlite3_last_insert_rowid", "p") # const char *sqlite3_errmsg(sqlite3*); sqlite3_errmsg = sq3.func("s", "sqlite3_errmsg", "p") -# Too recent -##const char *sqlite3_errstr(int); -# sqlite3_errstr = sq3.func("s", "sqlite3_errstr", "i") - SQLITE_OK = 0 SQLITE_ERROR = 1 @@ -51,6 +57,11 @@ SQLITE_BLOB = 4 SQLITE_NULL = 5 +SQLITE_CONFIG_URI = 17 + +# For compatibility with CPython sqlite3 driver +LEGACY_TRANSACTION_CONTROL = -1 + class Error(Exception): pass @@ -61,79 +72,142 @@ def check_error(db, s): raise Error(s, sqlite3_errmsg(db)) +def get_ptr_size(): + return uctypes.sizeof({"ptr": (0 | uctypes.PTR, uctypes.PTR)}) + + +def __prepare_stmt(db, sql): + # Prepares a statement + stmt_ptr = bytes(get_ptr_size()) + res = sqlite3_prepare(db, sql, -1, stmt_ptr, None) + check_error(db, res) + return int.from_bytes(stmt_ptr, sys.byteorder) + +def __exec_stmt(db, sql): + # Prepares, executes, and finalizes a statement + stmt = __prepare_stmt(db, sql) + sqlite3_step(stmt) + res = sqlite3_finalize(stmt) + check_error(db, res) + +def __is_dml(sql): + # Checks if a sql query is a DML, as these get a BEGIN in LEGACY_TRANSACTION_CONTROL + for dml in ["INSERT", "DELETE", "UPDATE", "MERGE"]: + if dml in sql.upper(): + return True + return False + + class Connections: - def __init__(self, h): - self.h = h + def __init__(self, db, isolation_level, autocommit): + self.db = db + self.isolation_level = isolation_level + self.autocommit = autocommit + + def commit(self): + if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "COMMIT") + elif self.autocommit == False: + __exec_stmt(self.db, "COMMIT") + __exec_stmt(self.db, "BEGIN") + + def rollback(self): + if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "ROLLBACK") + elif self.autocommit == False: + __exec_stmt(self.db, "ROLLBACK") + __exec_stmt(self.db, "BEGIN") def cursor(self): - return Cursor(self.h) + return Cursor(self.db, self.isolation_level, self.autocommit) def close(self): - s = sqlite3_close(self.h) - check_error(self.h, s) + if self.db: + if self.autocommit == False and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "ROLLBACK") + + res = sqlite3_close(self.db) + check_error(self.db, res) + self.db = None class Cursor: - def __init__(self, h): - self.h = h - self.stmnt = None + def __init__(self, db, isolation_level, autocommit): + self.db = db + self.isolation_level = isolation_level + self.autocommit = autocommit + self.stmt = None + + def __quote(val): + if isinstance(val, str): + return "'%s'" % val + return str(val) def execute(self, sql, params=None): + if self.stmt: + # If there is an existing statement, finalize that to free it + res = sqlite3_finalize(self.stmt) + check_error(self.db, res) + if params: - params = [quote(v) for v in params] + params = [self.__quote(v) for v in params] sql = sql % tuple(params) - print(sql) - b = bytearray(4) - s = sqlite3_prepare(self.h, sql, -1, b, None) - check_error(self.h, s) - self.stmnt = int.from_bytes(b, sys.byteorder) - # print("stmnt", self.stmnt) - self.num_cols = sqlite3_column_count(self.stmnt) - # print("num_cols", self.num_cols) - # If it's not select, actually execute it here - # num_cols == 0 for statements which don't return data (=> modify it) + + if __is_dml(sql) and self.autocommit == LEGACY_TRANSACTION_CONTROL and sqlite3_get_autocommit(self.db): + # For compatibility with CPython, add functionality for their default transaction + # behavior. Changing autocommit from LEGACY_TRANSACTION_CONTROL will remove this + __exec_stmt(self.db, "BEGIN " + self.isolation_level) + + self.stmt = __prepare_stmt(self.db, sql) + self.num_cols = sqlite3_column_count(self.stmt) + if not self.num_cols: v = self.fetchone() + # If it's not select, actually execute it here + # num_cols == 0 for statements which don't return data (=> modify it) assert v is None - self.lastrowid = sqlite3_last_insert_rowid(self.h) + self.lastrowid = sqlite3_last_insert_rowid(self.db) def close(self): - s = sqlite3_finalize(self.stmnt) - check_error(self.h, s) + if self.stmt: + res = sqlite3_finalize(self.stmt) + check_error(self.db, res) + self.stmt = None - def make_row(self): + def __make_row(self): res = [] for i in range(self.num_cols): - t = sqlite3_column_type(self.stmnt, i) - # print("type", t) + t = sqlite3_column_type(self.stmt, i) if t == SQLITE_INTEGER: - res.append(sqlite3_column_int(self.stmnt, i)) + res.append(sqlite3_column_int(self.stmt, i)) elif t == SQLITE_FLOAT: - res.append(sqlite3_column_double(self.stmnt, i)) + res.append(sqlite3_column_double(self.stmt, i)) elif t == SQLITE_TEXT: - res.append(sqlite3_column_text(self.stmnt, i)) + res.append(sqlite3_column_text(self.stmt, i)) else: raise NotImplementedError return tuple(res) def fetchone(self): - res = sqlite3_step(self.stmnt) - # print("step:", res) + res = sqlite3_step(self.stmt) if res == SQLITE_DONE: return None if res == SQLITE_ROW: - return self.make_row() - check_error(self.h, res) + return self.__make_row() + check_error(self.db, res) + + +def connect(fname, uri=False, isolation_level="", autocommit=LEGACY_TRANSACTION_CONTROL): + if isolation_level not in [None, "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE"]: + raise Error("Invalid option for isolation level") + sqlite3_config(SQLITE_CONFIG_URI, int(uri)) -def connect(fname): - b = bytearray(4) - sqlite3_open(fname, b) - h = int.from_bytes(b, sys.byteorder) - return Connections(h) + sqlite_ptr = bytes(get_ptr_size()) + sqlite3_open(fname, sqlite_ptr) + db = int.from_bytes(sqlite_ptr, sys.byteorder) + if autocommit == False: + __exec_stmt(db, "BEGIN") -def quote(val): - if isinstance(val, str): - return "'%s'" % val - return str(val) + return Connections(db, isolation_level, autocommit) diff --git a/unix-ffi/sqlite3/test_sqlite3.py b/unix-ffi/sqlite3/test_sqlite3.py index 39dc07549..b168f18ff 100644 --- a/unix-ffi/sqlite3/test_sqlite3.py +++ b/unix-ffi/sqlite3/test_sqlite3.py @@ -17,3 +17,6 @@ assert row == e assert expected == [] + +cur.close() +conn.close() diff --git a/unix-ffi/sqlite3/test_sqlite3_2.py b/unix-ffi/sqlite3/test_sqlite3_2.py index 68a2abb86..515f865c3 100644 --- a/unix-ffi/sqlite3/test_sqlite3_2.py +++ b/unix-ffi/sqlite3/test_sqlite3_2.py @@ -10,3 +10,6 @@ cur.execute("SELECT * FROM foo") assert cur.fetchone() == (42,) assert cur.fetchone() is None + +cur.close() +conn.close() diff --git a/unix-ffi/sqlite3/test_sqlite3_3.py b/unix-ffi/sqlite3/test_sqlite3_3.py new file mode 100644 index 000000000..0a6fefc97 --- /dev/null +++ b/unix-ffi/sqlite3/test_sqlite3_3.py @@ -0,0 +1,42 @@ +import sqlite3 + + +def test_autocommit(): + conn = sqlite3.connect(":memory:", autocommit=True) + + # First cursor creates table and inserts value (DML) + cur = conn.cursor() + cur.execute("CREATE TABLE foo(a int)") + cur.execute("INSERT INTO foo VALUES (42)") + cur.close() + + # Second cursor fetches 42 due to the autocommit + cur = conn.cursor() + cur.execute("SELECT * FROM foo") + assert cur.fetchone() == (42,) + assert cur.fetchone() is None + + cur.close() + conn.close() + +def test_manual(): + conn = sqlite3.connect(":memory:", autocommit=False) + + # First cursor creates table, insert rolls back + cur = conn.cursor() + cur.execute("CREATE TABLE foo(a int)") + conn.commit() + cur.execute("INSERT INTO foo VALUES (42)") + cur.close() + conn.rollback() + + # Second connection fetches nothing due to the rollback + cur = conn.cursor() + cur.execute("SELECT * FROM foo") + assert cur.fetchone() is None + + cur.close() + conn.close() + +test_autocommit() +test_manual() diff --git a/unix-ffi/time/manifest.py b/unix-ffi/time/manifest.py index d13942cd6..d1ff709a4 100644 --- a/unix-ffi/time/manifest.py +++ b/unix-ffi/time/manifest.py @@ -1,5 +1,5 @@ metadata(version="0.5.0") -require("ffilib", unix_ffi=True) +require("ffilib") module("time.py") diff --git a/unix-ffi/time/time.py b/unix-ffi/time/time.py index 075d904f5..319228dc8 100644 --- a/unix-ffi/time/time.py +++ b/unix-ffi/time/time.py @@ -1,6 +1,6 @@ from utime import * -from ucollections import namedtuple -import ustruct +from collections import namedtuple +import struct import uctypes import ffi import ffilib @@ -34,13 +34,13 @@ def _tuple_to_c_tm(t): - return ustruct.pack( + return struct.pack( "@iiiiiiiii", t[5], t[4], t[3], t[2], t[1] - 1, t[0] - 1900, (t[6] + 1) % 7, t[7] - 1, t[8] ) def _c_tm_to_tuple(tm): - t = ustruct.unpack("@iiiiiiiii", tm) + t = struct.unpack("@iiiiiiiii", tm) return _struct_time( t[5] + 1900, t[4] + 1, t[3], t[2], t[1], t[0], (t[6] - 1) % 7, t[7] + 1, t[8] ) @@ -64,7 +64,7 @@ def localtime(t=None): t = time() t = int(t) - a = ustruct.pack("l", t) + a = struct.pack("l", t) tm_p = localtime_(a) return _c_tm_to_tuple(uctypes.bytearray_at(tm_p, 36)) @@ -74,7 +74,7 @@ def gmtime(t=None): t = time() t = int(t) - a = ustruct.pack("l", t) + a = struct.pack("l", t) tm_p = gmtime_(a) return _c_tm_to_tuple(uctypes.bytearray_at(tm_p, 36)) diff --git a/unix-ffi/timeit/manifest.py b/unix-ffi/timeit/manifest.py index 94b6a5fe9..ea13af331 100644 --- a/unix-ffi/timeit/manifest.py +++ b/unix-ffi/timeit/manifest.py @@ -1,9 +1,9 @@ metadata(version="3.3.4") -require("getopt", unix_ffi=True) +require("getopt") require("itertools") # require("linecache") TODO -require("time", unix_ffi=True) +require("time") require("traceback") module("timeit.py") diff --git a/unix-ffi/ucurses/manifest.py b/unix-ffi/ucurses/manifest.py index 50648033e..8ec2675a5 100644 --- a/unix-ffi/ucurses/manifest.py +++ b/unix-ffi/ucurses/manifest.py @@ -2,8 +2,8 @@ # Originally written by Paul Sokolovsky. -require("os", unix_ffi=True) -require("tty", unix_ffi=True) -require("select", unix_ffi=True) +require("os") +require("tty") +require("select") package("ucurses") diff --git a/unix-ffi/urllib.parse/manifest.py b/unix-ffi/urllib.parse/manifest.py index 7023883f4..94109b134 100644 --- a/unix-ffi/urllib.parse/manifest.py +++ b/unix-ffi/urllib.parse/manifest.py @@ -1,6 +1,6 @@ metadata(version="0.5.2") -require("re", unix_ffi=True) +require("re") require("collections") require("collections-defaultdict")