Merge pull request #230 from elfenpiff/iox2-224-implement-placement-n…

…ew-for-containers

[#224] implement placement new for containers

.github/workflows/build-test.yml

@@ -89,7 +89,8 @@ jobs:
         rust-toolchain: stable
         check-and-install-cmd: cargo-nextest --version > /dev/null || cargo install cargo-nextest --locked
         print-version-cmd: cargo-nextest --version
-        cache-key: cache-1-${{ runner.os }}-cargo-nextest
+        # increment cache-N-${{}} if a new nextest version is required
+        cache-key: cache-2-${{ runner.os }}-cargo-nextest
         artifact-bin-name: cargo-nextest
         artifact-upload-name: ${{ runner.os }}-cargo-nextest
 
@@ -139,7 +140,47 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
-        toolchain: [stable, 1.75.0, beta, nightly]
+        toolchain: [stable, 1.75.0]
+        mode: 
+          - name: "release"
+            arg: "--release"
+          - name: "debug"
+            arg: ""
+    timeout-minutes: 60
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: ${{ matrix.toolchain }}
+          components: rustfmt, clippy
+
+      - name: Download artifact cargo-nextest
+        uses: ./.github/actions/download-cached-rust-tool
+        with:
+          artifact-bin-name: cargo-nextest
+          artifact-upload-name: ${{ runner.os }}-cargo-nextest
+
+      - name: Prepare system
+        run: ${{ matrix.os == 'windows-latest' && 'internal\scripts\ci_prepare_windows.bat' || ( matrix.os == 'ubuntu-latest' && './internal/scripts/ci_prepare_ubuntu.sh' || 'uname -a' ) }}
+
+      - name: Run cargo build
+        run: cargo build --workspace --all-targets ${{ matrix.mode.arg }}
+
+      - name: Run cargo nextest
+        run: cargo nextest run --workspace --no-fail-fast ${{ matrix.mode.arg }}
+
+  x86_64_unstable:
+    needs: [preflight-check, static-code-analysis, cargo-nextest]
+    if: ${{ needs.changes.outputs.source-code == 'true' }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-latest]
+        toolchain: [beta, nightly]
         mode: 
           - name: "release"
             arg: "--release"
@@ -213,6 +254,8 @@ jobs:
       - uses: vmactions/freebsd-vm@v1
         with:
           release: ${{ matrix.freebsd_version }}
+          mem: 8192
+          copyback: false
           run: |
             ./internal/scripts/ci_prepare_freebsd.sh
             export PATH=$PATH:$HOME/.cargo/bin

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "iceoryx2-bb/lock-free/",
     "iceoryx2-bb/threadsafe/",
     "iceoryx2-bb/container",
+    "iceoryx2-bb/derive-macros",
     "iceoryx2-bb/elementary",
     "iceoryx2-bb/log",
     "iceoryx2-bb/memory",
@@ -40,6 +41,7 @@ version = "0.3.0"
 iceoryx2-bb-threadsafe = { version = "0.3.0", path = "iceoryx2-bb/threadsafe/" }
 iceoryx2-bb-lock-free = { version = "0.3.0", path = "iceoryx2-bb/lock-free/" }
 iceoryx2-bb-container = { version = "0.3.0", path = "iceoryx2-bb/container/" }
+iceoryx2-bb-derive-macros = { version = "0.3.0", path = "iceoryx2-bb/derive-macros/" }
 iceoryx2-bb-elementary = { version = "0.3.0", path = "iceoryx2-bb/elementary/" }
 iceoryx2-bb-log = { version = "0.3.0", path = "iceoryx2-bb/log/" }
 iceoryx2-bb-memory = { version = "0.3.0", path = "iceoryx2-bb/memory/" }
@@ -68,8 +70,11 @@ log = { version = "0.4.21" }
 once_cell = { version = "1.19.0" }
 ouroboros = { version = "0.18.4" }
 pin-init = { version = "0.2.0" }
+proc-macro2 = { version = "1.0.84" }
+quote = { version = "1.0.36" }
 serde = { version = "1.0.203", features = ["derive"] }
 sha1_smol = { version = "1.0.0" }
+syn = { version = "2.0.66", features = ["full"] }
 termsize = { version = "0.1.6" }
 tiny-fn = { version = "0.1.5" }
 toml = { version = "0.8.13" }

FAQ.md

@@ -24,6 +24,14 @@ Use the feature flag `enforce_32bit_rwlock_atomic` which enforces 32-bit atomics
 targets at the cost of the lock-free guarantee. Meaning, when an application crashes at the wrong
 point in time it can lead to a system deadlock.
 
+## My Transmission Type Is Too Large, Encounter Stack Overflow On Initialization.
+
+Take a look at the
+[complex data types example](examples/rust/complex_data_types).
+
+In this example the `PlacementDefault` trait is introduced that allows in place initialization
+and solves the stack overflow issue when the data type is larger than the available stack size.
+
 ## Application does not remove services/ports on shutdown or several application restarts lead to port count exceeded
 
 The structs of iceoryx2 need to be able to cleanup all resources when they

doc/release-notes/iceoryx2-unreleased.md

@@ -44,7 +44,7 @@
     * Introduce `IoxAtomic` that supports up to 128bit atomics on 32-bit architecture with a ReadWriteLock
     * add CI targets to officially support 32-bit
  * Example that demonstrates publish-subscribe communication with dynamic data [#205](https://github.com/eclipse-iceoryx/iceoryx2/issues/205)
-
+ * Introduce `PlacementNew` trait and derive proc-macro [#224](https://github.com/eclipse-iceoryx/iceoryx2/issues/224)
 
 ### Bugfixes
 

examples/README.md

@@ -38,7 +38,7 @@ they interact and exchange data.
 
 | Name | Description |
 |------|-------------|
-| [complex data types](rust/complex_data_types) | Send zero-copy compatible versions of `Vec`, `String`, .... |
+| [complex data types](rust/complex_data_types) | Send zero-copy compatible versions of `Vec` and `String`. Introduces `PlacementDefault` trait for large data types to perform an in place initialization where otherwise a stack overflow would be encountered.|
 | [discovery](rust/discovery) | List all available services in a system. |
 | [docker](rust/docker) | Communicate between different docker containers and the host. |
 | [event](rust/event) | Exchanging event signals between multiple processes.|

examples/rust/complex_data_types/complex_data_types.rs

@@ -17,20 +17,24 @@ use iceoryx2_bb_container::{
     byte_string::FixedSizeByteString, queue::FixedSizeQueue, vec::FixedSizeVec,
 };
 
-#[derive(Debug, Default)]
+// For both data types we derive from PlacementDefault to allow in memory initialization
+// without any copy. Avoids stack overflows when data type is larger than the available stack.
+#[derive(Debug, Default, PlacementDefault)]
 #[repr(C)]
 pub struct ComplexData {
     name: FixedSizeByteString<4>,
     data: FixedSizeVec<u64, 4>,
 }
 
-#[derive(Debug, Default)]
+// For both data types we derive from PlacementDefault to allow in memory initialization
+// without any copy. Avoids stack overflows when data type is larger than the available stack.
+#[derive(Debug, Default, PlacementDefault)]
 #[repr(C)]
 pub struct ComplexDataType {
     plain_old_data: u64,
     text: FixedSizeByteString<8>,
     vec_of_data: FixedSizeVec<u64, 4>,
-    vec_of_complex_data: FixedSizeVec<ComplexData, 4>,
+    vec_of_complex_data: FixedSizeVec<ComplexData, 404857>,
     a_queue_of_things: FixedSizeQueue<FixedSizeByteString<4>, 2>,
 }
 
@@ -50,10 +54,15 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut counter = 0;
 
     while let Iox2Event::Tick = Iox2::wait(CYCLE_TIME) {
-        // acquire and send out sample
-        let mut sample = publisher.loan()?;
-        let payload = sample.payload_mut();
+        // ComplexDataType as a size of over 30MB, we need to perform a placement new
+        // otherwise we will encounter a stack overflow in debug builds.
+        // Therefore, we acquire an uninitialized sample, use the PlacementDefault
+        // trait to initialize ComplexDataType in place and then populate it with data.
+        let mut sample = publisher.loan_uninit()?;
+        unsafe { ComplexDataType::placement_default(sample.payload_mut().as_mut_ptr()) };
+        let mut sample = unsafe { sample.assume_init() };
 
+        let payload = sample.payload_mut();
         payload.plain_old_data = counter;
         payload.text = FixedSizeByteString::from_bytes(b"hello")?;
         payload.vec_of_data.push(counter);

iceoryx2-bb/container/Cargo.toml

@@ -13,6 +13,7 @@ version = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+iceoryx2-bb-derive-macros = { workspace = true }
 iceoryx2-bb-elementary = { workspace = true }
 iceoryx2-bb-log = { workspace = true }
 iceoryx2-pal-concurrency-sync = { workspace = true }

iceoryx2-bb/container/src/byte_string.rs

@@ -37,6 +37,8 @@ use std::{
     ops::{Deref, DerefMut},
 };
 
+use iceoryx2_bb_derive_macros::PlacementDefault;
+use iceoryx2_bb_elementary::placement_default::PlacementDefault;
 use iceoryx2_bb_log::{fail, fatal_panic};
 
 /// Returns the length of a string
@@ -71,12 +73,12 @@ impl std::fmt::Display for FixedSizeByteStringModificationError {
 impl std::error::Error for FixedSizeByteStringModificationError {}
 
 /// Relocatable string with compile time fixed size capacity.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, PlacementDefault)]
 #[repr(C)]
 pub struct FixedSizeByteString<const CAPACITY: usize> {
     len: usize,
     data: [MaybeUninit<u8>; CAPACITY],
-    _terminator: u8,
+    terminator: u8,
 }
 
 unsafe impl<const CAPACITY: usize> Send for FixedSizeByteString<CAPACITY> {}
@@ -222,7 +224,7 @@ impl<const CAPACITY: usize> FixedSizeByteString<CAPACITY> {
         let mut new_self = Self {
             len: 0,
             data: unsafe { MaybeUninit::uninit().assume_init() },
-            _terminator: 0,
+            terminator: 0,
         };
         new_self.data[0] = MaybeUninit::new(0);
         new_self

iceoryx2-bb/container/src/queue.rs

@@ -99,6 +99,7 @@ use iceoryx2_bb_elementary::allocator::{AllocationError, BaseAllocator};
 use iceoryx2_bb_elementary::math::align_to;
 use iceoryx2_bb_elementary::math::unaligned_mem_size;
 use iceoryx2_bb_elementary::owning_pointer::OwningPointer;
+use iceoryx2_bb_elementary::placement_default::PlacementDefault;
 use iceoryx2_bb_elementary::pointer_trait::PointerTrait;
 pub use iceoryx2_bb_elementary::relocatable_container::RelocatableContainer;
 use iceoryx2_bb_elementary::relocatable_ptr::RelocatablePointer;
@@ -397,15 +398,17 @@ pub struct FixedSizeQueue<T, const CAPACITY: usize> {
     _data: [MaybeUninit<T>; CAPACITY],
 }
 
+impl<T, const CAPACITY: usize> PlacementDefault for FixedSizeQueue<T, CAPACITY> {
+    unsafe fn placement_default(ptr: *mut Self) {
+        let state_ptr = core::ptr::addr_of_mut!((*ptr).state);
+        state_ptr.write(Self::initialize_state());
+    }
+}
+
 impl<T, const CAPACITY: usize> Default for FixedSizeQueue<T, CAPACITY> {
     fn default() -> Self {
         Self {
-            state: unsafe {
-                RelocatableQueue::new(
-                    CAPACITY,
-                    align_to::<MaybeUninit<T>>(std::mem::size_of::<RelocatableQueue<T>>()) as isize,
-                )
-            },
+            state: Self::initialize_state(),
             _data: unsafe { MaybeUninit::uninit().assume_init() },
         }
     }
@@ -415,6 +418,15 @@ unsafe impl<T: Send, const CAPACITY: usize> Send for FixedSizeQueue<T, CAPACITY>
 unsafe impl<T: Sync, const CAPACITY: usize> Sync for FixedSizeQueue<T, CAPACITY> {}
 
 impl<T, const CAPACITY: usize> FixedSizeQueue<T, CAPACITY> {
+    fn initialize_state() -> RelocatableQueue<T> {
+        unsafe {
+            RelocatableQueue::new(
+                CAPACITY,
+                align_to::<MaybeUninit<T>>(std::mem::size_of::<RelocatableQueue<T>>()) as isize,
+            )
+        }
+    }
+
     /// Creates a new queue.
     pub fn new() -> Self {
         Self::default()

iceoryx2-bb/container/src/vec.rs

@@ -79,6 +79,7 @@ use std::{alloc::Layout, mem::MaybeUninit, ops::Deref, ops::DerefMut, sync::atom
 
 use iceoryx2_bb_elementary::{
     math::{align_to, unaligned_mem_size},
+    placement_default::PlacementDefault,
     pointer_trait::PointerTrait,
     relocatable_container::RelocatableContainer,
     relocatable_ptr::RelocatablePointer,
@@ -330,15 +331,17 @@ pub struct FixedSizeVec<T, const CAPACITY: usize> {
     _data: [MaybeUninit<T>; CAPACITY],
 }
 
+impl<T, const CAPACITY: usize> PlacementDefault for FixedSizeVec<T, CAPACITY> {
+    unsafe fn placement_default(ptr: *mut Self) {
+        let state_ptr = core::ptr::addr_of_mut!((*ptr).state);
+        state_ptr.write(Self::initialize_state())
+    }
+}
+
 impl<T, const CAPACITY: usize> Default for FixedSizeVec<T, CAPACITY> {
     fn default() -> Self {
         Self {
-            state: unsafe {
-                RelocatableVec::new(
-                    CAPACITY,
-                    align_to::<MaybeUninit<T>>(std::mem::size_of::<RelocatableVec<T>>()) as isize,
-                )
-            },
+            state: Self::initialize_state(),
             _data: core::array::from_fn(|_| MaybeUninit::uninit()),
         }
     }
@@ -377,6 +380,15 @@ impl<T: Clone, const CAPACITY: usize> Clone for FixedSizeVec<T, CAPACITY> {
 unsafe impl<T: Send, const CAPACITY: usize> Send for FixedSizeVec<T, CAPACITY> {}
 
 impl<T, const CAPACITY: usize> FixedSizeVec<T, CAPACITY> {
+    fn initialize_state() -> RelocatableVec<T> {
+        unsafe {
+            RelocatableVec::new(
+                CAPACITY,
+                align_to::<MaybeUninit<T>>(std::mem::size_of::<RelocatableVec<T>>()) as isize,
+            )
+        }
+    }
+
     /// Creates a new vector.
     pub fn new() -> Self {
         Self::default()

iceoryx2-bb/container/tests/byte_string_tests.rs

@@ -17,7 +17,8 @@ mod fixed_size_byte_string {
     };
 
     use iceoryx2_bb_container::byte_string::*;
-    use iceoryx2_bb_testing::assert_that;
+    use iceoryx2_bb_elementary::placement_default::PlacementDefault;
+    use iceoryx2_bb_testing::{assert_that, memory::RawMemory};
     use std::collections::hash_map::DefaultHasher;
 
     const SUT_CAPACITY: usize = 129;
@@ -500,4 +501,14 @@ mod fixed_size_byte_string {
         let mut sut = Sut::from_bytes_truncated(b"Who did eat the last unicorn?");
         sut.remove_range(48, 12);
     }
+
+    #[test]
+    fn placement_default_works() {
+        let mut sut = RawMemory::<Sut>::new_filled(0xff);
+        unsafe { Sut::placement_default(sut.as_mut_ptr()) };
+        assert_that!(unsafe {sut.assume_init()}, len 0);
+
+        assert_that!(unsafe { sut.assume_init_mut() }.push_bytes(b"hello"), is_ok);
+        assert_that!(unsafe {sut.assume_init()}.as_bytes(), eq b"hello");
+    }
 }

iceoryx2-bb/container/tests/queue_tests.rs

@@ -12,8 +12,10 @@
 
 mod queue {
     use iceoryx2_bb_container::queue::*;
-    use iceoryx2_bb_elementary::bump_allocator::BumpAllocator;
-    use iceoryx2_bb_testing::{assert_that, lifetime_tracker::LifetimeTracker};
+    use iceoryx2_bb_elementary::{
+        bump_allocator::BumpAllocator, placement_default::PlacementDefault,
+    };
+    use iceoryx2_bb_testing::{assert_that, lifetime_tracker::LifetimeTracker, memory::RawMemory};
 
     const SUT_CAPACITY: usize = 128;
     type Sut = FixedSizeQueue<usize, SUT_CAPACITY>;
@@ -291,4 +293,18 @@ mod queue {
             assert_that!(unsafe { sut.get_unchecked(i) }, eq i * 3 + 2);
         }
     }
+
+    #[test]
+    fn placement_default_works() {
+        type Sut = FixedSizeQueue<usize, SUT_CAPACITY>;
+        let mut sut = RawMemory::<Sut>::new_filled(0xff);
+        unsafe { Sut::placement_default(sut.as_mut_ptr()) };
+
+        assert_that!(unsafe {sut.assume_init()}, len 0);
+        assert_that!(unsafe {sut.assume_init_mut()}.push(123), eq true);
+        assert_that!(unsafe {sut.assume_init_mut()}.push(456), eq true);
+
+        assert_that!(unsafe {sut.assume_init_mut()}.pop(), eq Some(123));
+        assert_that!(unsafe {sut.assume_init_mut()}.pop(), eq Some(456));
+    }
 }

iceoryx2-bb/container/tests/vec_tests.rs

@@ -12,9 +12,11 @@
 
 use iceoryx2_bb_container::vec::*;
 use iceoryx2_bb_elementary::bump_allocator::BumpAllocator;
+use iceoryx2_bb_elementary::placement_default::PlacementDefault;
 use iceoryx2_bb_elementary::relocatable_container::RelocatableContainer;
 use iceoryx2_bb_testing::assert_that;
 use iceoryx2_bb_testing::lifetime_tracker::LifetimeTracker;
+use iceoryx2_bb_testing::memory::RawMemory;
 
 const SUT_CAPACITY: usize = 128;
 type Sut = FixedSizeVec<usize, SUT_CAPACITY>;
@@ -243,3 +245,17 @@ fn fixed_size_vec_pop_releases_ownership() {
         assert_that!(LifetimeTracker::number_of_living_instances(), eq i);
     }
 }
+
+#[test]
+fn placement_default_works() {
+    type Sut = FixedSizeVec<usize, SUT_CAPACITY>;
+    let mut sut = RawMemory::<Sut>::new_filled(0xff);
+    unsafe { Sut::placement_default(sut.as_mut_ptr()) };
+
+    assert_that!(unsafe { sut.assume_init()}, len 0);
+    assert_that!(unsafe { sut.assume_init_mut()}.push(123), eq true);
+    assert_that!(unsafe { sut.assume_init_mut()}.push(456), eq true);
+
+    assert_that!(unsafe { sut.assume_init_mut()}.pop(), eq Some(456));
+    assert_that!(unsafe { sut.assume_init_mut()}.pop(), eq Some(123));
+}