From 9b88c9097ea6327c6fbb68792e4fed36006c457c Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Sun, 12 Nov 2023 14:06:55 +1100 Subject: [PATCH 1/9] Added rust tests --- Cargo.lock | 40 +++++++- jcan/Cargo.toml | 3 + jcan/justfile | 2 + jcan/src/lib.rs | 38 ++++++-- jcan/tests/harness.rs | 59 ++++++++++++ jcan/tests/tests.rs | 209 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 337 insertions(+), 14 deletions(-) create mode 100644 jcan/justfile create mode 100644 jcan/tests/harness.rs create mode 100644 jcan/tests/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 268a02d..09d9485 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,6 +203,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "jcan" version = "0.2.0" @@ -212,8 +218,11 @@ dependencies = [ "cxx-build", "embedded-can", "env_logger", + "lazy_static", "log", "nix", + "serde", + "serde_json", "socketcan", ] @@ -226,6 +235,12 @@ dependencies = [ "pyo3-build-config", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.147" @@ -521,6 +536,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + [[package]] name = "scopeguard" version = "1.2.0" @@ -543,24 +564,35 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.183" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", "syn 2.0.28", ] +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "smallvec" version = "1.11.0" diff --git a/jcan/Cargo.toml b/jcan/Cargo.toml index c5fb762..5bc3456 100644 --- a/jcan/Cargo.toml +++ b/jcan/Cargo.toml @@ -17,6 +17,9 @@ socketcan = "2.0.0" cxx = "1.0" log = "0.4.17" env_logger = "0.10.0" +serde_json = "1.0.108" +serde = { version = "1.0.192", features = ["derive"] } +lazy_static = "1.4.0" [build-dependencies] cxx-build = "1.0" diff --git a/jcan/justfile b/jcan/justfile new file mode 100644 index 0000000..f679d72 --- /dev/null +++ b/jcan/justfile @@ -0,0 +1,2 @@ +test: + cargo test -- --show-output --test-threads 1 diff --git a/jcan/src/lib.rs b/jcan/src/lib.rs index 1c3c5a1..1f0934d 100644 --- a/jcan/src/lib.rs +++ b/jcan/src/lib.rs @@ -1,9 +1,10 @@ extern crate socketcan; -use log::{debug, error, warn}; +use log::{debug, info, error, warn}; use embedded_can::{ExtendedId, Frame as EmbeddedFrame, Id, StandardId}; use socketcan::{CanFilter, CanFrame, CanSocket, Socket}; use std::sync::mpsc; +use std::sync::mpsc::TrySendError; use std::sync::Arc; use std::sync::atomic::Ordering; use std::thread; @@ -13,7 +14,7 @@ use std::time::Duration; pub mod ffi { #[cxx_name = "Frame"] - #[derive(Clone)] + #[derive(Clone,PartialEq,Eq)] pub struct JFrame { id: u32, data: Vec, @@ -184,15 +185,21 @@ impl JBus { let frame: ffi::JFrame = frame.into(); // Send the frame to the channel - match tx.send(frame) { + match tx.try_send(frame) { Ok(_) => { // All good! debug!("jcan_recv_thread queued a frame for next spin()"); } Err(e) => { - // This error can only occur if there are no receivers - // If this happens, the main thread has closed, and we should also close - warn!("jcan_recv_thread failed to queue frame: {}",e); + // This error can only occur if there are no receivers, or the buffer is full. + match e { + TrySendError::Full(_) => { + debug!("jcan_recv_thread dropped a received frame, buffer is full"); + }, + TrySendError::Disconnected(_) => { + break; + }, + } } } } @@ -212,6 +219,13 @@ impl JBus { // This means the socket is closed, break the loop. // Check the os error code _ => { + // Check if .close() was called + if !socket_opened_clone.load(Ordering::Relaxed) { + info!("jcan_recv_thread closed"); + break; + } + + // Print different logs match e.raw_os_error() { Some(19) => { // Network is down @@ -535,7 +549,7 @@ pub fn new_jbus() -> Result, std::io::Error> { match env_logger::builder().filter_level(log::LevelFilter::Warn).try_init() { Ok(_) => {} Err(_) => { - warn!("env_logger already initialised"); + info!("env_logger already initialised"); } } @@ -610,15 +624,19 @@ impl ffi::JFrame { // Implement Display for JFrame, used for Rust only impl std::fmt::Display for ffi::JFrame { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - // Prints in the same format as CANDUMP + // OLD: Prints in the same format as CANDUMP // e.g: vcan0 123 [2] 10 20 + // NEW: Prints in the same format as cansend input + // e.g: cansend vcan0 123#AABBCC + let mut s = String::new(); - s.push_str(&format!("0x{:03X} [{}] ", self.id, self.data.len())); + //s.push_str(&format!("0x{:03X} [{}] ", self.id, self.data.len())); + s.push_str(&format!("{:03X}#", self.id)); for byte in self.data.iter() { - s.push_str(&format!("{:02X} ", byte)); + s.push_str(&format!("{:02X}", byte)); } write!(f, "{}", s).unwrap(); diff --git a/jcan/tests/harness.rs b/jcan/tests/harness.rs new file mode 100644 index 0000000..2c3df6c --- /dev/null +++ b/jcan/tests/harness.rs @@ -0,0 +1,59 @@ +// For constructing frames +extern crate jcan; +use jcan::{ffi::JFrame}; + +// For running tests +use std::process::Command; +use serde::{Deserialize}; +use serde_json; +use std::sync::Mutex; +use lazy_static::lazy_static; + +#[derive(Debug, Deserialize)] +pub struct Link { + ifname: String, +} + +lazy_static! { + static ref TEST_MUTEX: Mutex<()> = Mutex::new(()); +} + +#[allow(dead_code)] +pub fn add_vcan() -> String { + // Grab the TEST_MUTEX to prevent threads getting the interfaces muddled + let _guard = TEST_MUTEX.lock().unwrap(); + + // Add a new vcan interface (vcan) + assert!(Command::new("sudo").args(&["ip","link","add","type","vcan"]).status().unwrap().success()); + + // Get a list of virtual CAN interfaces, as JSON format. + let ifs = Command::new("sudo").args(&["ip","-j","link","show","type","vcan"]).output().expect("Failed to list vcan links"); + + let if_json = match String::from_utf8(ifs.stdout) { + Ok(v) => v, + Err(e) => panic!("Invalid UTF-8 sequence: {}", e), + }; + + // Decode JSON into a Vector of Link objects + let ifs_obj : Vec = serde_json::from_str(&if_json).expect("Failed to parse JSON"); + // Get the name of the last link available + let iface = &ifs_obj.last().unwrap().ifname; + + println!("Created {}",iface); + assert!(Command::new("sudo").args(&["ip","link","set","up",&iface.to_string()]).status().unwrap().success()); + + iface.to_string() +} + +#[allow(dead_code)] +pub fn del_vcan(iface : String) { + assert!(Command::new("sudo").args(&["ip","link","del",&iface]).status().unwrap().success()); + println!("Deleted {}",iface); +} + +#[allow(dead_code)] +pub fn cansend_vcan(iface : String, frame : &JFrame) { + // Uses the string representation of JFrame, e.g 123#AABBCCDD + assert!(Command::new("sudo").args(&["cansend", &iface, &frame.to_string()]).status().unwrap().success()); +} + diff --git a/jcan/tests/tests.rs b/jcan/tests/tests.rs new file mode 100644 index 0000000..8d02690 --- /dev/null +++ b/jcan/tests/tests.rs @@ -0,0 +1,209 @@ +#![allow(non_snake_case)] + +// Import jcan +extern crate jcan; +use jcan::*; + +// Import testing harness +mod harness; +use harness::{*}; + +#[test] +fn test_vcan_harness() { + let iface = add_vcan(); + del_vcan(iface); +} + +#[test] +fn test_bus_open_close() { + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + assert!(bus.open(iface.clone(), 2, 256).is_ok()); + + assert!(bus.close().is_ok()); + del_vcan(iface); +} + +#[test] +fn test_send() { + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + assert!(bus.open(iface.clone(), 2, 256).is_ok()); + + let frame = new_jframe(0x123,(&[0xA,0xB,0xC,0xD]).to_vec()).expect("Failed to build Frame"); + let _ = bus.send(frame); + + assert!(bus.close().is_ok()); + del_vcan(iface); +} + +#[test] +fn test_send_receive() { + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + assert!(bus.open(iface.clone(), 2, 256).is_ok()); + + // Build a frame to send using 'cansend' tool + let frame = new_jframe(0x123,(&[0xA,0xA]).to_vec()).expect("Failed to build Frame"); + + // Send a frame + cansend_vcan(iface.clone(), &frame); + + // Receive a frame, comparing it to the expected one + let rx = bus.receive().expect("Failed to receive Frame"); + println!("{}",rx); + assert!(rx == frame); + + assert!(bus.close().is_ok()); + del_vcan(iface); +} + +#[test] +fn test_id_filter() { + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + + // Apply a filter to only receive frames with + // ID == 0x210 ONLY! + bus.set_id_filter((&[ 0x210 ]).to_vec()).expect("Failed to set Frame ID filter"); + bus.open(iface.clone(), 2, 256).expect("Failed to open Bus"); + + // Build two frames to send - the first one should be ignored! + let frame1 = new_jframe(0x123,(&[0xAA,0xBB]).to_vec()).expect("Failed to build Frame"); + let frame2 = new_jframe(0x210,(&[0xCC,0xDD]).to_vec()).expect("Failed to build Frame"); + + // Send the two frames + cansend_vcan(iface.clone(), &frame1); + cansend_vcan(iface.clone(), &frame2); + + // Receive a frame, comparing it to the expected one + let rx = bus.receive().expect("Failed to receive Frame"); + println!("{}",rx); + + assert!(rx == frame2); + + assert!(bus.close().is_ok()); + del_vcan(iface); +} + +#[test] +fn test_receive_timeout() { + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + assert!(bus.open(iface.clone(), 2, 256).is_ok()); + + // Receive a frame, comparing it to the expected one + let rx = bus.receive_with_timeout_millis(300); + + assert!(rx.is_err()); + + assert!(bus.close().is_ok()); + del_vcan(iface); +} + + +#[test] +fn test_receive_buffer() { + // Send 100 frames using cansend, and then receive them. + // PASS if we received all 100 frames. + let N = 30u16; + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + bus.open(iface.clone(), 2, 256).expect("Failed to open Bus"); + + let frame1 = new_jframe(0x123,(&[0xAA,0xBB]).to_vec()).expect("Failed to build Frame"); + + for _ in 0..N { + cansend_vcan(iface.clone(), &frame1); + } + + // Receives from the Vec buffer directly + let rxs = bus.receive_from_thread_buffer().expect("Failed to receive from thread buffer"); + println!("Received {} frames", rxs.len()); + + assert!(rxs.len() == N as usize); + assert!(bus.close().is_ok()); + + del_vcan(iface); +} + +#[test] +fn test_receive_buffer_alternate() { + // Send 100 frames using cansend, and then receive them. + // PASS if we received all 100 frames. + // This test uses bus.receive() in a loop, instead of receive_from_thread_buffer(). + + let N = 30u16; + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + bus.open(iface.clone(), 2, 256).expect("Failed to open Bus"); + + let frame1 = new_jframe(0x123,(&[0xAA,0xBB]).to_vec()).expect("Failed to build Frame"); + + for _ in 0..N { + cansend_vcan(iface.clone(), &frame1); + } + println!("Sent {} frames", N); + + // Receives from the Vec buffer directly + let mut rxs : Vec = Vec::new(); + + for _ in 0..N { + let rx = bus.receive_with_timeout_millis(100); + match rx { + Ok(r) => rxs.push(r), + Err(_) => break, + } + } + println!("Received {} frames", rxs.len()); + + assert!(rxs.len() == N as usize); + assert!(bus.close().is_ok()); + + del_vcan(iface); +} + +#[test] +fn test_receive_buffer_overflow() { + // Send 500 frames using cansend, dropping 250 of them, because our receive buffer is + // only 250 frames long. + + let N = 100u16; + let B = 50u16; + + let iface = add_vcan(); + + let mut bus = new_jbus().unwrap(); + bus.open(iface.clone(), 2, B).expect("Failed to open Bus"); + + let frame1 = new_jframe(0x123,(&[0xAA,0xBB]).to_vec()).expect("Failed to build Frame"); + + for _ in 0..N { + cansend_vcan(iface.clone(), &frame1); + } + println!("Sent {} frames", N); + + // Receives from the Vec buffer directly + let mut rxs : Vec = Vec::new(); + + for _ in 0..N { + let rx = bus.receive_with_timeout_millis(100); + match rx { + Ok(r) => rxs.push(r), + Err(_) => break, + } + } + println!("Received {} frames", rxs.len()); + + assert!(rxs.len() == B as usize); + assert!(bus.close().is_ok()); + + del_vcan(iface); +} From 3dd774eecd8a7efe19cc150922ab837b2e97aec1 Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Fri, 19 Jan 2024 23:03:29 +1100 Subject: [PATCH 2/9] Working toward some pytest functions, and better nix-shell experience. --- jcan/src/lib.rs | 7 ++-- jcan_python/jcan/jcan_python.pyi | 53 ++++++++++++++++++++++++++++++ jcan_python/jcan/py.typed | 0 jcan_python/pyproject.toml | 3 -- jcan_python/setup.py | 11 +++---- jcan_python/tests.py | 20 ++++++++++++ shell.nix | 56 +++++++++++++++++++++----------- 7 files changed, 118 insertions(+), 32 deletions(-) create mode 100644 jcan_python/jcan/jcan_python.pyi create mode 100644 jcan_python/jcan/py.typed delete mode 100644 jcan_python/pyproject.toml create mode 100644 jcan_python/tests.py diff --git a/jcan/src/lib.rs b/jcan/src/lib.rs index 1f0934d..3b2feda 100644 --- a/jcan/src/lib.rs +++ b/jcan/src/lib.rs @@ -544,9 +544,10 @@ impl JBus { // Builder for JBus, used to create C++ instances of the opaque JBus type // Takes in a String interface pub fn new_jbus() -> Result, std::io::Error> { - // Initialise the logger, if it hasn't already been initialised - // This is done by the first call to new_jbus() - match env_logger::builder().filter_level(log::LevelFilter::Warn).try_init() { + // make a logger which shows Warnings by default, + // but can show other levels by setting + // JCAN_LOG=info/debug/fatal/error, etc + match env_logger::builder().filter_level(log::LevelFilter::Warn).parse_env("JCAN_LOG").try_init() { Ok(_) => {} Err(_) => { info!("env_logger already initialised"); diff --git a/jcan_python/jcan/jcan_python.pyi b/jcan_python/jcan/jcan_python.pyi new file mode 100644 index 0000000..3509879 --- /dev/null +++ b/jcan_python/jcan/jcan_python.pyi @@ -0,0 +1,53 @@ +# EXAMPLE OF PEP484 stub-type definition. +# class Car: +# """ +# A class representing a car. + +# :param body_type: the name of body type, e.g. hatchback, sedan +# :param horsepower: power of the engine in horsepower +# """ +# def __init__(self, body_type: str, horsepower: int) -> None: ... + +# @classmethod +# def from_unique_name(cls, name: str) -> 'Car': +# """ +# Creates a Car based on unique name + +# :param name: model name of a car to be created +# :return: a Car instance with default data +# """ + +# def best_color(self) -> str: +# """ +# Gets the best color for the car. + +# :return: the name of the color our great algorithm thinks is the best for this car +# """ + +# Below are some hand-crafted stub-types for the JCAN python package. +# This means you get nice auto-completion and type-analysis, even though it's a static library. +# This file is based off the contents of ./jcan_python/src/lib.rs +from typing import List, Optional, Self, Callable + +class Frame: + def __init__(self, id: int, data: List[int]) -> None: ... + def __str__(self) -> str: ... + def id(self) -> int: ... + def data(self) -> List[int]: ... + +class Bus: + def open(self, interface: str, tx_queue_len: int = 2, rx_queue_len: int = 256) -> None: ... + def close(self) -> None: ... + def is_open(self) -> bool: ... + def callbacks_enabled(self) -> bool: ... + def set_callbacks_enabled(self, mode: bool) -> None: ... + def receive(self) -> Optional[Frame]: ... + def receive_with_timeout(self, timeout_ms: int) -> Optional[Frame]: ... + def send(self, frame: Frame) -> None: ... + def drop_buffered_frames(self) -> None: ... + def set_id_filter(self, allowed_ids: List[int]) -> None: ... + def set_id_filter_mask(self, allowed: int, allowed_mask: int) -> None: ... + def receive_from_thread_buffer(self) -> Optional[List[Frame]]: ... + def add_callback(self, frame_id: int, callback: Callable[[Frame], None]) -> None: ... + def spin(self) -> None: ... + diff --git a/jcan_python/jcan/py.typed b/jcan_python/jcan/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/jcan_python/pyproject.toml b/jcan_python/pyproject.toml deleted file mode 100644 index f1f99db..0000000 --- a/jcan_python/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel", "setuptools-rust", "toml"] - diff --git a/jcan_python/setup.py b/jcan_python/setup.py index 451a76e..50d96aa 100644 --- a/jcan_python/setup.py +++ b/jcan_python/setup.py @@ -32,11 +32,8 @@ def get_version() -> str: ) ], include_package_data=True, - # Requirements - # install_requires=[ - # "setuptools-rust", - # ], - # setup_requires=[ - # "setuptools-rust", - # ], + + # setuptools_rust is required to build, + # and pytest is required to test + #setup_requires=["setuptools-rust", "pytest"], ) diff --git a/jcan_python/tests.py b/jcan_python/tests.py new file mode 100644 index 0000000..e4f1a6f --- /dev/null +++ b/jcan_python/tests.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +# pytest tests for JCAN +# Leigh Oliver, January 2024 +# (I finished my degree last year <3) + +import pytest +from jcan import Bus, Frame + +def test_open_bus(): + bus = Bus() + bus.open("vcan0") + bus.close() + +def test_send_receive(): + bus = Bus() + bus.open("vcan0") + + frame = Frame(0x100, [1,2,3,4,5,6,7,8]) + bus.send(frame) \ No newline at end of file diff --git a/shell.nix b/shell.nix index 4a214fb..63a4212 100644 --- a/shell.nix +++ b/shell.nix @@ -1,12 +1,20 @@ # shell.nix -# This installs the jcan library (C++ and Python) from the source code, -# by building it with Nix. You are then dropped into a bash shell, where these libraries are -# available to use in your application. -# -# This shell is also useful for JCAN development. -# -{ pkgs ? import {} }: +# This shell is useful for JCAN development. let + # rust development environment stuff + # makes rust-analyzer work, and uses nightly rust + rustOverlay = builtins.fetchTarball "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"; + + pkgs = import { + overlays = [ (import rustOverlay) ]; + }; + + rust = pkgs.rust-bin.nightly.latest.default.override { + extensions = [ + "rust-src" # for rust-analyzer + ]; + }; + # rust/C++ library, and python wrapper jcan = pkgs.callPackage ./jcan.nix {}; jcan-python = pkgs.python3Packages.callPackage ./jcan_python.nix {}; @@ -15,22 +23,32 @@ let mk-vcan = pkgs.writeShellScriptBin "mk-vcan" (builtins.readFile ./utils/mk-vcan.sh); rm-vcan = pkgs.writeShellScriptBin "rm-vcan" (builtins.readFile ./utils/rm-vcan.sh); in -pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - rustup +pkgs.mkShell rec { + buildInputs = [ rust ] ++ (with pkgs; [ + bacon + can-utils cargo + gcc jcan jcan-python - python3 - python310Packages.pip - can-utils mk-vcan - rm-vcan + pkg-config podman - ]; + python3 + python3Packages.pip + python3Packages.pytest + rm-vcan + rust-analyzer + rustup + stdenv.cc + ]); + + LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}"; - shellHook = - '' - echo "Run 'mk-vcan' and 'rm-vcan' to use virtual CAN interfaces!" - ''; + shellHook = '' + export PS1="''${debian_chroot:+($debian_chroot)}\[\033[01;39m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\W\[\033[00m\]\$ " + export PS1="(jcan)$PS1" + export LD_LIBRARY_PATH="''${LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" + echo "Run 'mk-vcan' and 'rm-vcan' to use virtual CAN interfaces!" + ''; } From 755373914e7a16bf639978a12c15691aafb14a9e Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Fri, 19 Jan 2024 23:08:56 +1100 Subject: [PATCH 3/9] Updated MANIFEST.in --- jcan_python/MANIFEST.in | 2 ++ jcan_python/pyproject.toml | 3 +++ jcan_python/setup.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 jcan_python/pyproject.toml diff --git a/jcan_python/MANIFEST.in b/jcan_python/MANIFEST.in index 7c68298..18527fe 100644 --- a/jcan_python/MANIFEST.in +++ b/jcan_python/MANIFEST.in @@ -1,2 +1,4 @@ include Cargo.toml +include ./jcan/jcan_python.pyi +include ./jcan/py.typed recursive-include src * diff --git a/jcan_python/pyproject.toml b/jcan_python/pyproject.toml new file mode 100644 index 0000000..f1f99db --- /dev/null +++ b/jcan_python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel", "setuptools-rust", "toml"] + diff --git a/jcan_python/setup.py b/jcan_python/setup.py index 50d96aa..eae2dc1 100644 --- a/jcan_python/setup.py +++ b/jcan_python/setup.py @@ -35,5 +35,5 @@ def get_version() -> str: # setuptools_rust is required to build, # and pytest is required to test - #setup_requires=["setuptools-rust", "pytest"], + setup_requires=["setuptools-rust", "pytest", "toml"], ) From e42d9ca314a17198dcd4969d6012777c486bd439 Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Fri, 19 Jan 2024 23:28:38 +1100 Subject: [PATCH 4/9] Nix nix nix --- jcan/Cargo.toml | 2 +- jcan_python.nix | 17 +++++++---------- jcan_python/Cargo.toml | 2 +- jcan_python/setup.py | 6 +++++- shell.nix | 2 ++ 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/jcan/Cargo.toml b/jcan/Cargo.toml index 5bc3456..f49cc59 100644 --- a/jcan/Cargo.toml +++ b/jcan/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jcan" -version = "0.2.0" +version = "0.2.4" edition = "2021" publish = false links = "jcan" diff --git a/jcan_python.nix b/jcan_python.nix index 45c2281..a990ff6 100644 --- a/jcan_python.nix +++ b/jcan_python.nix @@ -1,17 +1,12 @@ { pkgs ? import {} , lib ? pkgs.lib , buildPythonPackage ? pkgs.python3Packages.buildPythonPackage -, rustPlatform ? pkgs.rustPlatform -, cargo ? pkgs.cargo -, rustc ? pkgs.rustc -, setuptools-rust ? pkgs.python3Packages.setuptools-rust -, toml ? pkgs.python3Packages.toml }: buildPythonPackage rec { name = "jcan-python"; - doCheck = false; + doCheck = true; outputs = [ "out" ]; @@ -20,16 +15,18 @@ buildPythonPackage rec { dontPatchELF = true; - cargoDeps = rustPlatform.importCargoLock { + cargoDeps = pkgs.rustPlatform.importCargoLock { lockFile = ./Cargo.lock; }; - nativeBuildInputs = [ + nativeBuildInputs = with pkgs; [ cargo rustPlatform.cargoSetupHook rustc - setuptools-rust - toml + python3Packages.setuptools-rust + python3Packages.toml + python3Packages.pip + python3Packages.pytest ]; postPatch = '' diff --git a/jcan_python/Cargo.toml b/jcan_python/Cargo.toml index 1e692ea..6e2bb9f 100644 --- a/jcan_python/Cargo.toml +++ b/jcan_python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jcan_python" -version = "0.2.0" +version = "0.2.4" edition = "2021" publish = false diff --git a/jcan_python/setup.py b/jcan_python/setup.py index eae2dc1..30d3739 100644 --- a/jcan_python/setup.py +++ b/jcan_python/setup.py @@ -33,7 +33,11 @@ def get_version() -> str: ], include_package_data=True, + package_data={ + "jcan": ["py.typed", "jcan_python.pyi"], + }, + # setuptools_rust is required to build, # and pytest is required to test - setup_requires=["setuptools-rust", "pytest", "toml"], + # setup_requires=["setuptools-rust", "pytest", "toml"], ) diff --git a/shell.nix b/shell.nix index 63a4212..71bbfd8 100644 --- a/shell.nix +++ b/shell.nix @@ -37,6 +37,8 @@ pkgs.mkShell rec { python3 python3Packages.pip python3Packages.pytest + python3Packages.setuptools-rust + python3Packages.toml rm-vcan rust-analyzer rustup From bd8c89c76a89662cd4f7096c4d4b3d088be26e1b Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Fri, 19 Jan 2024 23:42:44 +1100 Subject: [PATCH 5/9] Updated type hints --- Cargo.lock | 4 +- jcan_python/jcan/jcan_python.pyi | 99 ++++++++++++++++++++++---------- shell.nix | 17 ++++-- 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09d9485..113975b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -211,7 +211,7 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jcan" -version = "0.2.0" +version = "0.2.4" dependencies = [ "cc", "cxx", @@ -228,7 +228,7 @@ dependencies = [ [[package]] name = "jcan_python" -version = "0.2.0" +version = "0.2.4" dependencies = [ "jcan", "pyo3", diff --git a/jcan_python/jcan/jcan_python.pyi b/jcan_python/jcan/jcan_python.pyi index 3509879..e4608a2 100644 --- a/jcan_python/jcan/jcan_python.pyi +++ b/jcan_python/jcan/jcan_python.pyi @@ -1,53 +1,90 @@ -# EXAMPLE OF PEP484 stub-type definition. -# class Car: -# """ -# A class representing a car. - -# :param body_type: the name of body type, e.g. hatchback, sedan -# :param horsepower: power of the engine in horsepower -# """ -# def __init__(self, body_type: str, horsepower: int) -> None: ... - -# @classmethod -# def from_unique_name(cls, name: str) -> 'Car': -# """ -# Creates a Car based on unique name - -# :param name: model name of a car to be created -# :return: a Car instance with default data -# """ - -# def best_color(self) -> str: -# """ -# Gets the best color for the car. - -# :return: the name of the color our great algorithm thinks is the best for this car -# """ - # Below are some hand-crafted stub-types for the JCAN python package. # This means you get nice auto-completion and type-analysis, even though it's a static library. # This file is based off the contents of ./jcan_python/src/lib.rs -from typing import List, Optional, Self, Callable +from typing import List, Optional, Self, Callable, Union class Frame: - def __init__(self, id: int, data: List[int]) -> None: ... + def __init__(self, id: int, data: List[Union[int,float]]) -> None: ... + """ + :param: id: The 11-bit CAN ID of the frame. + :param: data: The data bytes of the frame, which will be cast to uint8's by JCAN library - so be careful to double check the values! + """ def __str__(self) -> str: ... def id(self) -> int: ... + """ + :return: The 11-bit CAN ID of the frame. + """ def data(self) -> List[int]: ... + """ + :return: The data bytes of the frame. + """ class Bus: def open(self, interface: str, tx_queue_len: int = 2, rx_queue_len: int = 256) -> None: ... + """ + :param: interface: The name of the CAN interface to open, e.g. "vcan0". + :param: tx_queue_len: The length of the internal transmit queue, after which send() will block. + :param: rx_queue_len: The length of the internal receive queue, after which older Frames will be dropped. + """ def close(self) -> None: ... + """ + Closes the CAN interface. + """ def is_open(self) -> bool: ... + """ + :return: True if the CAN interface is open, False otherwise. + """ def callbacks_enabled(self) -> bool: ... + """ + :return: True if callbacks are enabled, False otherwise. + """ def set_callbacks_enabled(self, mode: bool) -> None: ... - def receive(self) -> Optional[Frame]: ... - def receive_with_timeout(self, timeout_ms: int) -> Optional[Frame]: ... + """ + :param: mode: True to enable callbacks, False to disable. + """ + def receive(self) -> Frame: ... + """ + Blocks until a frame is received, then returns it. + :return: The received frame. + """ + def receive_with_timeout(self, timeout_ms: int) -> Frame: ... + """ + Blocks until a frame is received, or the timeout expires, then returns it. + :param: timeout_ms: The timeout in milliseconds. + :return: The received frame, or None if the timeout expired. + """ def send(self, frame: Frame) -> None: ... + """ + Sends a frame, blocking until it is queued for transmission on the TX queue. + :param: frame: The frame to send. + """ def drop_buffered_frames(self) -> None: ... + """ + Drop all frames in the RX queue. + """ def set_id_filter(self, allowed_ids: List[int]) -> None: ... + """ + Set a filter for the CAN ID's that will be received, and for which callbacks will be called. + :param: allowed_ids: A list of allowed CAN ID's. + """ def set_id_filter_mask(self, allowed: int, allowed_mask: int) -> None: ... - def receive_from_thread_buffer(self) -> Optional[List[Frame]]: ... + """ + Set a filter for the CAN ID's that will be received, and for which callbacks will be called. + :param: allowed: The base allowed ID value. + :param: allowed_mask: The mask for bits of the allowed ID, which are to be checked. + """ + def receive_from_thread_buffer(self) -> List[Frame]: ... + """ + :return: A list of frames that are currently in the internal receive buffer. + """ def add_callback(self, frame_id: int, callback: Callable[[Frame], None]) -> None: ... + """ + Add a callback for a specific CAN ID. + :param: frame_id: The CAN ID for which the callback will be called. + :param: callback: The callback function, which will be called with the received frame as an argument. + """ def spin(self) -> None: ... + """ + Needs to be called periodically to process received frames and call callbacks. + """ diff --git a/shell.nix b/shell.nix index 71bbfd8..bfab3a0 100644 --- a/shell.nix +++ b/shell.nix @@ -15,9 +15,15 @@ let ]; }; - # rust/C++ library, and python wrapper - jcan = pkgs.callPackage ./jcan.nix {}; + #jcan = pkgs.callPackage ./jcan.nix {}; jcan-python = pkgs.python3Packages.callPackage ./jcan_python.nix {}; + + # useful script to build into a local ./result/ path. + build-jcan-python = pkgs.writeShellScriptBin "build-jcan-python" '' + #!/usr/bin/env bash + cargo clean + nix-build -E 'let pkgs = import {}; in pkgs.python3Packages.callPackage ./jcan_python.nix {}' + ''; # utility scripts mk-vcan = pkgs.writeShellScriptBin "mk-vcan" (builtins.readFile ./utils/mk-vcan.sh); @@ -29,9 +35,8 @@ pkgs.mkShell rec { can-utils cargo gcc - jcan + #jcan jcan-python - mk-vcan pkg-config podman python3 @@ -39,10 +44,12 @@ pkgs.mkShell rec { python3Packages.pytest python3Packages.setuptools-rust python3Packages.toml - rm-vcan rust-analyzer rustup stdenv.cc + rm-vcan + mk-vcan + build-jcan-python ]); LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}"; From 91a134785f22c2054a8ab04204f974160040857a Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Fri, 19 Jan 2024 23:46:45 +1100 Subject: [PATCH 6/9] pytest is tricky when VCAN needs sudo --- jcan_python/tests.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/jcan_python/tests.py b/jcan_python/tests.py index e4f1a6f..0f70054 100644 --- a/jcan_python/tests.py +++ b/jcan_python/tests.py @@ -5,16 +5,5 @@ # (I finished my degree last year <3) import pytest +import subprocess from jcan import Bus, Frame - -def test_open_bus(): - bus = Bus() - bus.open("vcan0") - bus.close() - -def test_send_receive(): - bus = Bus() - bus.open("vcan0") - - frame = Frame(0x100, [1,2,3,4,5,6,7,8]) - bus.send(frame) \ No newline at end of file From 2259742f04bcd822c91d618fb87b369c5a054467 Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Fri, 19 Jan 2024 23:59:21 +1100 Subject: [PATCH 7/9] cargo clean before building jcan_python --- jcan_python.nix | 15 +++++++++++++-- jcan_python/jcan/jcan_python.pyi | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/jcan_python.nix b/jcan_python.nix index a990ff6..9eec240 100644 --- a/jcan_python.nix +++ b/jcan_python.nix @@ -7,12 +7,23 @@ buildPythonPackage rec { name = "jcan-python"; doCheck = true; + pythonImportsCheck = [ "jcan" ]; - outputs = [ "out" ]; src = lib.cleanSource ./.; sourceRoot = "source/jcan_python"; + preBuild = '' + # not cleaning causes some issues due to permissions, + # for some reason. + cargo clean + + #ls $src + #exit 0 + #rm -rf ./target/ + ''; + + outputs = [ "out" ]; dontPatchELF = true; cargoDeps = pkgs.rustPlatform.importCargoLock { @@ -20,8 +31,8 @@ buildPythonPackage rec { }; nativeBuildInputs = with pkgs; [ - cargo rustPlatform.cargoSetupHook + cargo rustc python3Packages.setuptools-rust python3Packages.toml diff --git a/jcan_python/jcan/jcan_python.pyi b/jcan_python/jcan/jcan_python.pyi index e4608a2..70c6c3f 100644 --- a/jcan_python/jcan/jcan_python.pyi +++ b/jcan_python/jcan/jcan_python.pyi @@ -20,6 +20,7 @@ class Frame: """ class Bus: + def __init__(self) -> None: ... def open(self, interface: str, tx_queue_len: int = 2, rx_queue_len: int = 256) -> None: ... """ :param: interface: The name of the CAN interface to open, e.g. "vcan0". From 4f752034a9de1c6c876b12e58cb1ce8152d034e0 Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Sat, 20 Jan 2024 00:02:16 +1100 Subject: [PATCH 8/9] Wow forgot how cursed cross-build.nix is --- cross-build.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cross-build.nix b/cross-build.nix index 9c3e6b9..9b6f3ed 100644 --- a/cross-build.nix +++ b/cross-build.nix @@ -83,11 +83,11 @@ in targetPkgs = pkgs: [ pkgs.rustup pkgs.cargo - pkgs.python3 + pkgs.python310 pkgs.python310Packages.pip pkgs.python310Packages.wheel pkgs.python310Packages.setuptools-rust - pkgs.python3Packages.toml + pkgs.python310Packages.toml pkgs.docker #pkgs.podman pkgs.hostname From e7141af063f5b0554b91564637ac27d3a8dcc5c0 Mon Sep 17 00:00:00 2001 From: Leigh Oliver Date: Sat, 20 Jan 2024 00:33:41 +1100 Subject: [PATCH 9/9] nix crimes --- README.md | 2 +- cross-build.nix | 121 +++++++++--------------------------------------- jcan/src/lib.rs | 2 +- jcan_python.nix | 8 ++-- shell.nix | 2 +- 5 files changed, 27 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index d18c69b..8232fba 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ git clone https://github.com/leighleighleigh/JCAN # in a development environment: nix-shell -# To build cross-compiled C++ libraries for x86 and ARM64, +# (BROKEN) To build cross-compiled C++ libraries for x86 and ARM64, # and produce Python 3.8+ compatible wheels: nix-shell cross-build.nix diff --git a/cross-build.nix b/cross-build.nix index 9b6f3ed..b95a120 100644 --- a/cross-build.nix +++ b/cross-build.nix @@ -1,103 +1,24 @@ -# cross-build.nix -# This shell uses the 'cross-rs' tool to cross-compile JCAN. -# Whilst this is done easily on Nix, through it's own cross-compilation support, -# it doesn't produce a python wheel which is useful on non-Nix systems. -# -# Hence, this shell is primarily designed to produce a Python wheel for release to PyPi. -# -{ pkgs ? import {} }: let - clean-script = pkgs.writeScript "clean.sh" '' - #!/usr/bin/env bash - - SCRIPT_DIR=$( cd -- "$( dirname -- "''${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - - # Remove jcan-python/dist, build, **.egg-info folders - rm -rf "''${SCRIPT_DIR}/out" - rm -rf "''${SCRIPT_DIR}/jcan_python/dist" - rm -rf "''${SCRIPT_DIR}/jcan_python/build" - rm -rf "''${SCRIPT_DIR}/jcan_python/jcan.egg-info" - - # Run cargo clean - cargo clean - ''; - - build-script = pkgs.writeScript "crossbuild.sh" '' - #!/usr/bin/env bash - SCRIPT_DIR=$( cd -- "$( dirname -- "''${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - - # This function takes a TARGET as an argument, and builds the library for that target - # It then moves the build artifacts to out///jcan/ - function build_target { - CROSSTARGET="''${1}" - export CROSS_CONTAINER_ENGINE=docker - - # Very important to clean, incase old crates for x86 are present - cargo clean - cross build --package jcan --target $CROSSTARGET --release - cross build --package scripts_postbuild --target $CROSSTARGET --release - - # python build uses a special pyo3 image - export CARGO=cross - export CARGO_BUILD_TARGET=''${CROSSTARGET} - export CROSS_CONFIG=''${SCRIPT_DIR}/jcan_python/Cross.toml - - cross build --package jcan_python --target $CROSSTARGET --release - - # Run setuptools-rust on jcan_python - cd jcan_python - rm -rf ./dist - rm -rf ./build - - # Change plat-name depending on CROSSTARGET - if [[ "''${CROSSTARGET}" == "aarch64-unknown-linux-gnu" ]]; - then - PLATNAME="manylinux2014_aarch64" - elif [[ "''${CROSSTARGET}" == "x86_64-unknown-linux-gnu" ]]; - then - PLATNAME="manylinux2014_x86_64" - else - echo "Unknown CROSSTARGET: ''${CROSSTARGET}" - exit 1 - fi - - python setup.py bdist_wheel --plat-name $PLATNAME --py-limited-api=cp38 || exit 1 - - cd .. - - # Copy the resulting wheels to out folder - mkdir -p out/python/ - cp -r jcan_python/dist/*.whl out/python/ - } - - # Build for aarch64 - build_target "aarch64-unknown-linux-gnu" - - # Build for x86_64 - build_target "x86_64-unknown-linux-gnu" - ''; -in -(pkgs.buildFHSEnv { - name = "jcan-cross-env"; - - targetPkgs = pkgs: [ - pkgs.rustup - pkgs.cargo - pkgs.python310 - pkgs.python310Packages.pip - pkgs.python310Packages.wheel - pkgs.python310Packages.setuptools-rust - pkgs.python310Packages.toml - pkgs.docker - #pkgs.podman - pkgs.hostname - pkgs.direnv + rustOverlay = builtins.fetchTarball "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"; + + pkgs = import { + overlays = [ (import rustOverlay) ]; + crossSystem = { + config = "aarch64-unknown-linux-gnu"; + }; + }; + + rust = pkgs.rust-bin.nightly.latest.default.override { + extensions = [ + "rust-src" # for rust-analyzer ]; + }; - runScript = pkgs.writeScript "init.sh" '' - export PYTHONPATH="/lib/python3.10/site-packages/" - bash ${clean-script} - bash ${build-script} - rm -rf ./target - ''; -}).env + jcan-python = pkgs.python3Packages.callPackage ./jcan_python.nix {pkgs=pkgs;}; +in +pkgs.mkShell { + buildInputs = [rust jcan-python]; + shellHook = '' + echo "hi" + ''; +} diff --git a/jcan/src/lib.rs b/jcan/src/lib.rs index 3b2feda..490e840 100644 --- a/jcan/src/lib.rs +++ b/jcan/src/lib.rs @@ -547,7 +547,7 @@ pub fn new_jbus() -> Result, std::io::Error> { // make a logger which shows Warnings by default, // but can show other levels by setting // JCAN_LOG=info/debug/fatal/error, etc - match env_logger::builder().filter_level(log::LevelFilter::Warn).parse_env("JCAN_LOG").try_init() { + match env_logger::builder().filter_level(log::LevelFilter::Error).parse_env("JCAN_LOG").try_init() { Ok(_) => {} Err(_) => { info!("env_logger already initialised"); diff --git a/jcan_python.nix b/jcan_python.nix index 9eec240..c69b88c 100644 --- a/jcan_python.nix +++ b/jcan_python.nix @@ -1,16 +1,14 @@ { pkgs ? import {} -, lib ? pkgs.lib -, buildPythonPackage ? pkgs.python3Packages.buildPythonPackage }: -buildPythonPackage rec { +pkgs.python3Packages.buildPythonPackage rec { name = "jcan-python"; doCheck = true; pythonImportsCheck = [ "jcan" ]; - src = lib.cleanSource ./.; + src = pkgs.lib.cleanSource ./.; sourceRoot = "source/jcan_python"; preBuild = '' @@ -30,7 +28,7 @@ buildPythonPackage rec { lockFile = ./Cargo.lock; }; - nativeBuildInputs = with pkgs; [ + buildInputs = with pkgs; [ rustPlatform.cargoSetupHook cargo rustc diff --git a/shell.nix b/shell.nix index bfab3a0..7595914 100644 --- a/shell.nix +++ b/shell.nix @@ -22,7 +22,7 @@ let build-jcan-python = pkgs.writeShellScriptBin "build-jcan-python" '' #!/usr/bin/env bash cargo clean - nix-build -E 'let pkgs = import {}; in pkgs.python3Packages.callPackage ./jcan_python.nix {}' + nix-build -E 'let pkgs = import {crossSystem={config="aarch64-unknown-linux-gnu";};}; in pkgs.python3Packages.callPackage ./jcan_python.nix {pkgs=pkgs;}' ''; # utility scripts